Skip to content

Commit

Permalink
some more work
Browse files Browse the repository at this point in the history 
TODO: we want to let LLVM know that Memory lengths and Array dims must be > 0
  • Loading branch information
@vtjnash
vtjnash committed Sep 18, 2023
1 parent 822bc73 commit a03a8df
Show file tree
Hide file tree
Showing 4 changed files with 93 additions and 67 deletions.
2 changes: 1 addition & 1 deletion base/array.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -973,7 +973,7 @@ end

## Iteration ##

iterate(A::Array, i=1) = (@inline; (i % UInt) - 1 < length(A) ? (@inbounds A[i], i + 1) : nothing)
iterate(A::Array, i=1) = (@inline; (i - 1)%UInt < length(A)%UInt ? (@inbounds A[i], i + 1) : nothing)

## Indexing: getindex ##

Expand Down
5 changes: 2 additions & 3 deletions base/essentials.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -783,13 +783,12 @@ eval(:(function getindex(A::Array, i::Int)
#=:inaccessiblememonly=#0x2, # set consistent_if_inaccessiblememonly
#=:noub=#false)))=#
@boundscheck (i - 1)%UInt < length(A)%UInt || throw_boundserror(A, i)
ref = memoryref(A.ref, i, false)
memoryrefget(ref, :not_atomic, false)
memoryrefget(memoryref(getfield(A, :ref), i, false), :not_atomic, false)
end))
# simple Array{Any} operations needed for bootstrap
function setindex!(A::Array{Any}, @nospecialize(x), i::Int)
@boundscheck (i - 1)%UInt < length(A)%UInt || throw_boundserror(A, i)
memoryrefset(memoryref(A.ref, i, false), x, :not_atomic, false)
memoryrefset(memoryref(getfield(A, :ref), i, false), x, :not_atomic, false)
end
@eval setindex!(A::Memory{Any}, @nospecialize(x), i::Int) = memoryrefset(memoryref(memoryref(A), i, $(Expr(:boundscheck))), x, :not_atomic, $(Expr(:boundscheck)))
#@eval setindex!(A::AtomicMemory{Any}, @nospecialize(x), order::Symbol, i::Int) = memoryrefset(memoryref(memoryref(A), i, $(Expr(:boundscheck))), x, order, false)
Expand Down
2 changes: 1 addition & 1 deletion base/genericmemory.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ end

## Iteration ##

iterate(A::Memory, i=1) = (@inline; (i % UInt) - 1 < length(A) ? (@inbounds A[i], i + 1) : nothing)
iterate(A::Memory, i=1) = (@inline; (i - 1)%UInt < length(A)%UInt ? (@inbounds A[i], i + 1) : nothing)

## Indexing: getindex ##

Expand Down
151 changes: 89 additions & 62 deletions src/cgutils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1912,14 +1912,32 @@ static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, j
emit_memcpy(ctx, intcast, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), data, jl_aliasinfo_t::fromTBAA(ctx, tbaa), nb, alignment);
}
else {
LoadInst *load = ctx.builder.CreateAlignedLoad(elty, data, Align(alignment), false);
load->setOrdering(Order);
if (isboxed)
maybe_mark_load_dereferenceable(load, true, jltype);
jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa);
ai.scope = MDNode::concatenate(aliasscope, ai.scope);
ai.decorateInst(load);
instr = load;
if (!isboxed && jl_is_genericmemoryref_type(jltype)) { // TODO: experiment with loading FCA as individual fields, so LLVM does not need to split them later
Value *fld0 = ctx.builder.CreateStructGEP(elty, data, 0);
LoadInst *load0 = ctx.builder.CreateAlignedLoad(elty->getStructElementType(0), fld0, Align(alignment), false);
load0->setOrdering(Order);
maybe_mark_load_dereferenceable(load0, true, sizeof(void*)*2, alignof(void*));
jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa);
ai.scope = MDNode::concatenate(aliasscope, ai.scope);
ai.decorateInst(load0);
Value *fld1 = ctx.builder.CreateStructGEP(elty, data, 1);
LoadInst *load1 = ctx.builder.CreateAlignedLoad(elty->getStructElementType(1), fld1, Align(alignment), false);
load1->setOrdering(Order);
ai.decorateInst(load1);
instr = Constant::getNullValue(elty);
instr = ctx.builder.CreateInsertValue(instr, load0, 0);
instr = ctx.builder.CreateInsertValue(instr, load1, 1);
}
else {
LoadInst *load = ctx.builder.CreateAlignedLoad(elty, data, Align(alignment), false);
load->setOrdering(Order);
if (isboxed)
maybe_mark_load_dereferenceable(load, true, jltype);
jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa);
ai.scope = MDNode::concatenate(aliasscope, ai.scope);
ai.decorateInst(load);
instr = load;
}
if (elty != realelty)
instr = ctx.builder.CreateTrunc(instr, realelty);
if (intcast) {
Expand Down Expand Up @@ -3992,8 +4010,15 @@ static jl_cgval_t emit_memoryref(jl_codectx_t &ctx, const jl_cgval_t &ref, jl_cg
}
#endif
boffset = ctx.builder.CreateMul(offset, elsz);
#if 0 // TODO: if opaque-pointers?
newdata = emit_bitcast(ctx, data, getInt8PtrTy(ctx.builder.getContext()));
newdata = ctx.builder.CreateGEP(getInt8Ty(ctx.builder.getContext()), newdata, boffset);
#else
Type *elty = isboxed ? ctx.types().T_prjlvalue : julia_type_to_llvm(ctx, jl_tparam1(ref.typ));
newdata = emit_bitcast(ctx, data, elty->getPointerTo(0));
newdata = ctx.builder.CreateInBoundsGEP(elty, newdata, offset);
(void)boffset; // LLVM is very bad at handling GEP with types different from the load
#endif
newdata = emit_bitcast(ctx, newdata, data->getType());
if (bc) {
BasicBlock *failBB, *endBB;
Expand Down Expand Up @@ -4057,61 +4082,63 @@ static Value *emit_memoryref_ptr(jl_codectx_t &ctx, const jl_cgval_t &ref, bool
Value *mem = CreateSimplifiedExtractValue(ctx, newref, 1);
// rebuild GEP on data, so that we manually hoist this gc_loaded_func call over it, back to the original load
// we should add this to llvm-julia-licm too, so we can attempt hoisting over PhiNodes too (which aren't defined yet here)
{
IRBuilder<>::InsertPointGuard resetIP(ctx.builder);
std::vector<GetElementPtrInst*> GEPlist;
while (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(data->stripPointerCastsSameRepresentation())) { // ignoring bitcast will not be required with opaque pointers
GEPlist.push_back(GEP);
data = GEP->getPointerOperand();
}
// Under the common assumption that data and mem will come from the same basic block and both will dominate the first GEP,
// manually hoist these right away to improve the IR structure. But tolerate the case where we could not verify that.
bool moveip = false;
//Instruction *defmem = dyn_cast<Instruction>(mem);
//if (Instruction *def = dyn_cast<Instruction>(data)) {
// if (defmem == nullptr || def->getParent() == defmem->getParent()) {
// if (defmem != nullptr && !defmem->comesBefore(def))
// def = defmem;
// ctx.builder.SetInsertPoint(def->getParent(), std::next(def->getIterator()));
// ctx.builder.SetCurrentDebugLocation(def->getDebugLoc());
// moveip = true;
// }
// else if (LoadInst *defbase = dyn_cast<LoadInst>(def)) {
// def = defbase;
// if (defmem != nullptr && !defmem->comesBefore(def))
// def = defmem;
// ctx.builder.SetInsertPoint(def->getParent(), std::next(def->getIterator()));
// ctx.builder.SetCurrentDebugLocation(def->getDebugLoc());
// moveip = true;
// }
//}
//// if we couldn't find the def Instruction, we might still be able to put it right before the GEP chain
//if (!moveip && !GEPlist.empty()) {
// Instruction *defgep = GEPlist[GEPlist.size() - 1];
// if (defmem == nullptr || defmem->getParent() == defgep->getParent()) {
// if (defmem == nullptr || defmem->comesBefore(defgep)) {
// ctx.builder.SetInsertPoint(GEPlist[GEPlist.size() - 1]);
// moveip = true;
// }
// }
//}
data = ctx.builder.CreateBitCast(data, ctx.types().T_pprjlvalue);
data = ctx.builder.CreateCall(prepare_call(gc_loaded_func), { mem, data });
if (!GEPlist.empty()) {
for (auto &GEP : make_range(GEPlist.rbegin(), GEPlist.rend())) {
if (moveip)
ctx.builder.SetInsertPoint(GEP);
data = ctx.builder.CreateBitCast(data, PointerType::get(GEP->getSourceElementType(), AS));
Instruction *GEP2 = GEP->clone();
GEP2->mutateType(PointerType::get(GEP->getResultElementType(), AS));
GEP2->setOperand(GetElementPtrInst::getPointerOperandIndex(), data);
ctx.builder.Insert(GEP2);
data = GEP2;
}
IRBuilder<>::InsertPointGuard resetIP(ctx.builder);
std::vector<GetElementPtrInst*> GEPlist;
data = data->stripPointerCastsSameRepresentation();
while (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(data)) { // ignoring bitcast will not be required with opaque pointers
GEPlist.push_back(GEP);
data = GEP->getPointerOperand()->stripPointerCastsSameRepresentation();
}
// Under the common assumption that data and mem will come from the same basic block and both will dominate the first GEP,
// manually hoist these right away to improve the IR structure. But tolerate the case where we could not verify that.
bool moveip = false;
//Instruction *defmem = dyn_cast<Instruction>(mem);
//if (Instruction *def = dyn_cast<Instruction>(data)) {
// if (defmem == nullptr || def->getParent() == defmem->getParent()) {
// if (defmem != nullptr && !defmem->comesBefore(def))
// def = defmem;
// ctx.builder.SetInsertPoint(def->getParent(), std::next(def->getIterator()));
// ctx.builder.SetCurrentDebugLocation(def->getDebugLoc());
// moveip = true;
// }
// else if (LoadInst *defbase = dyn_cast<LoadInst>(def)) {
// def = defbase;
// if (defmem != nullptr && !defmem->comesBefore(def))
// def = defmem;
// ctx.builder.SetInsertPoint(def->getParent(), std::next(def->getIterator()));
// ctx.builder.SetCurrentDebugLocation(def->getDebugLoc());
// moveip = true;
// }
//}
//// if we couldn't find the def Instruction, we might still be able to put it right before the GEP chain
//if (!moveip && !GEPlist.empty()) {
// Instruction *defgep = GEPlist[GEPlist.size() - 1];
// if (defmem == nullptr || defmem->getParent() == defgep->getParent()) {
// if (defmem == nullptr || defmem->comesBefore(defgep)) {
// ctx.builder.SetInsertPoint(GEPlist[GEPlist.size() - 1]);
// moveip = true;
// }
// }
//}
data = ctx.builder.CreateBitCast(data, ctx.types().T_pprjlvalue);
#if 0 // XXX(jwn): do not do this
(void)mem;
data = ctx.builder.CreateAddrSpaceCast(data, prepare_call(gc_loaded_func)->getReturnType());
#else
data = ctx.builder.CreateCall(prepare_call(gc_loaded_func), { mem, data });
#endif
if (!GEPlist.empty()) {
for (auto &GEP : make_range(GEPlist.rbegin(), GEPlist.rend())) {
if (moveip)
ctx.builder.SetInsertPoint(GEP);
data = ctx.builder.CreateBitCast(data, PointerType::get(GEP->getSourceElementType(), AS));
Instruction *GEP2 = GEP->clone();
GEP2->mutateType(PointerType::get(GEP->getResultElementType(), AS));
GEP2->setOperand(GetElementPtrInst::getPointerOperandIndex(), data);
ctx.builder.Insert(GEP2);
data = GEP2;
}
} // resetIP
if (isboxed)
data = ctx.builder.CreateBitCast(data, PointerType::get(ctx.types().T_prjlvalue, AS));
}
}
return data;
}
Expand Down

0 comments on commit a03a8df

Please sign in to comment.