diff --git a/base/atomics.jl b/base/atomics.jl index 38f8a0e243178..51c6964a13e16 100644 --- a/base/atomics.jl +++ b/base/atomics.jl @@ -345,20 +345,23 @@ for typ in atomictypes irt = Base.libllvm_version >= v"3.6" ? "$ilt, $ilt*" : "$ilt*" @eval getindex(x::Atomic{$typ}) = llvmcall($""" - %rv = load atomic $rt %0 acquire, align $(alignment(typ)) + %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %rv = load atomic $rt %ptr acquire, align $(alignment(typ)) ret $lt %rv """, $typ, Tuple{Ptr{$typ}}, unsafe_convert(Ptr{$typ}, x)) @eval setindex!(x::Atomic{$typ}, v::$typ) = llvmcall($""" - store atomic $lt %1, $lt* %0 release, align $(alignment(typ)) + %ptr = inttoptr i$WORD_SIZE %0 to $lt* + store atomic $lt %1, $lt* %ptr release, align $(alignment(typ)) ret void - """, Void, Tuple{Ptr{$typ},$typ}, unsafe_convert(Ptr{$typ}, x), v) + """, Void, Tuple{Ptr{$typ}, $typ}, unsafe_convert(Ptr{$typ}, x), v) # Note: atomic_cas! succeeded (i.e. it stored "new") if and only if the result is "cmp" if typ <: Integer @eval atomic_cas!(x::Atomic{$typ}, cmp::$typ, new::$typ) = llvmcall($""" - %rs = cmpxchg $lt* %0, $lt %1, $lt %2 acq_rel acquire + %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 acq_rel acquire %rv = extractvalue { $lt, i1 } %rs, 0 ret $lt %rv """, $typ, Tuple{Ptr{$typ},$typ,$typ}, @@ -366,7 +369,7 @@ for typ in atomictypes else @eval atomic_cas!(x::Atomic{$typ}, cmp::$typ, new::$typ) = llvmcall($""" - %iptr = bitcast $lt* %0 to $ilt* + %iptr = inttoptr i$WORD_SIZE %0 to $ilt* %icmp = bitcast $lt %1 to $ilt %inew = bitcast $lt %2 to $ilt %irs = cmpxchg $ilt* %iptr, $ilt %icmp, $ilt %inew acq_rel acquire @@ -387,14 +390,15 @@ for typ in atomictypes if typ <: Integer @eval $fn(x::Atomic{$typ}, v::$typ) = llvmcall($""" - %rv = atomicrmw $rmw $lt* %0, $lt %1 acq_rel + %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %rv = atomicrmw $rmw $lt* %ptr, $lt %1 acq_rel ret $lt %rv """, $typ, Tuple{Ptr{$typ}, $typ}, unsafe_convert(Ptr{$typ}, x), v) else rmwop == :xchg || continue @eval $fn(x::Atomic{$typ}, v::$typ) = llvmcall($""" - %iptr = bitcast $lt* %0 to $ilt* + %iptr = inttoptr i$WORD_SIZE %0 to $ilt* %ival = bitcast $lt %1 to $ilt %irv = atomicrmw $rmw $ilt* %iptr, $ilt %ival acq_rel %rv = bitcast $ilt %irv to $lt diff --git a/base/fastmath.jl b/base/fastmath.jl index b9c4f7db748b7..69b5cb7f758e5 100644 --- a/base/fastmath.jl +++ b/base/fastmath.jl @@ -296,34 +296,18 @@ asin_fast(x::FloatTypes) = asin(x) # explicit implementations -# FIXME: Change to `ccall((:sincos, libm))` when `Ref` calling convention can be -# stack allocated. @inline function sincos_fast(v::Float64) - return Base.llvmcall(""" - %f = bitcast i8 *%1 to void (double, double *, double *)* - %ps = alloca double - %pc = alloca double - call void %f(double %0, double *%ps, double *%pc) - %s = load double, double* %ps - %c = load double, double* %pc - %res0 = insertvalue [2 x double] undef, double %s, 0 - %res = insertvalue [2 x double] %res0, double %c, 1 - ret [2 x double] %res - """, Tuple{Float64,Float64}, Tuple{Float64,Ptr{Void}}, v, cglobal((:sincos, libm))) + s = Ref{Cdouble}() + c = Ref{Cdouble}() + ccall((:sincos, libm), Void, (Cdouble, Ptr{Cdouble}, Ptr{Cdouble}), v, s, c) + return (s[], c[]) end @inline function sincos_fast(v::Float32) - return Base.llvmcall(""" - %f = bitcast i8 *%1 to void (float, float *, float *)* - %ps = alloca float - %pc = alloca float - call void %f(float %0, float *%ps, float *%pc) - %s = load float, float* %ps - %c = load float, float* %pc - %res0 = insertvalue [2 x float] undef, float %s, 0 - %res = insertvalue [2 x float] %res0, float %c, 1 - ret [2 x float] %res - """, Tuple{Float32,Float32}, Tuple{Float32,Ptr{Void}}, v, cglobal((:sincosf, libm))) + s = Ref{Cfloat}() + c = Ref{Cfloat}() + ccall((:sincosf, libm), Void, (Cfloat, Ptr{Cfloat}, Ptr{Cfloat}), v, s, c) + return (s[], c[]) end @inline function sincos_fast(v::Float16) diff --git a/src/builtins.c b/src/builtins.c index 34b6430709384..0a5950e930d25 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -77,30 +77,40 @@ static int NOINLINE compare_svec(jl_svec_t *a, jl_svec_t *b) // See comment above for an explanation of NOINLINE. static int NOINLINE compare_fields(jl_value_t *a, jl_value_t *b, jl_datatype_t *dt) { - size_t nf = jl_datatype_nfields(dt); - for (size_t f=0; f < nf; f++) { + size_t f, nf = jl_datatype_nfields(dt); + for (f = 0; f < nf; f++) { size_t offs = jl_field_offset(dt, f); char *ao = (char*)jl_data_ptr(a) + offs; char *bo = (char*)jl_data_ptr(b) + offs; - int eq; if (jl_field_isptr(dt, f)) { jl_value_t *af = *(jl_value_t**)ao; jl_value_t *bf = *(jl_value_t**)bo; - if (af == bf) eq = 1; - else if (af==NULL || bf==NULL) eq = 0; - else eq = jl_egal(af, bf); + if (af != bf) { + if (af == NULL || bf == NULL) + return 0; + if (!jl_egal(af, bf)) + return 0; + } } else { jl_datatype_t *ft = (jl_datatype_t*)jl_field_type(dt, f); + if (jl_is_uniontype(ft)) { + uint8_t asel = ((uint8_t*)ao)[jl_field_size(dt, f) - 1]; + uint8_t bsel = ((uint8_t*)bo)[jl_field_size(dt, f) - 1]; + if (asel != bsel) + return 0; + ft = (jl_datatype_t*)jl_nth_union_component((jl_value_t*)ft, asel); + } if (!ft->layout->haspadding) { - eq = bits_equal(ao, bo, jl_field_size(dt, f)); + if (!bits_equal(ao, bo, jl_field_size(dt, f))) + return 0; } else { assert(jl_datatype_nfields(ft) > 0); - eq = compare_fields((jl_value_t*)ao, (jl_value_t*)bo, ft); + if (!compare_fields((jl_value_t*)ao, (jl_value_t*)bo, ft)) + return 0; } } - if (!eq) return 0; } return 1; } @@ -127,9 +137,11 @@ JL_DLLEXPORT int jl_egal(jl_value_t *a, jl_value_t *b) return 0; return !memcmp(jl_string_data(a), jl_string_data(b), l); } - if (dt->mutabl) return 0; + if (dt->mutabl) + return 0; size_t sz = jl_datatype_size(dt); - if (sz == 0) return 1; + if (sz == 0) + return 1; size_t nf = jl_datatype_nfields(dt); if (nf == 0) return bits_equal(jl_data_ptr(a), jl_data_ptr(b), sz); @@ -161,10 +173,10 @@ static uintptr_t bits_hash(void *b, size_t sz) static uintptr_t NOINLINE hash_svec(jl_svec_t *v) { uintptr_t h = 0; - size_t l = jl_svec_len(v); - for(size_t i = 0; i < l; i++) { - jl_value_t *x = jl_svecref(v,i); - uintptr_t u = x==NULL ? 0 : jl_object_id(x); + size_t i, l = jl_svec_len(v); + for (i = 0; i < l; i++) { + jl_value_t *x = jl_svecref(v, i); + uintptr_t u = (x == NULL) ? 0 : jl_object_id(x); h = bitmix(h, u); } return h; @@ -188,9 +200,11 @@ static uintptr_t jl_object_id_(jl_value_t *tv, jl_value_t *v) if (dt == jl_typename_type) return ((jl_typename_t*)v)->hash; #ifdef _P64 - if (v == jl_ANY_flag) return 0x31c472f68ee30bddULL; + if (v == jl_ANY_flag) + return 0x31c472f68ee30bddULL; #else - if (v == jl_ANY_flag) return 0x8ee30bdd; + if (v == jl_ANY_flag) + return 0x8ee30bdd; #endif if (dt == jl_string_type) { #ifdef _P64 @@ -199,24 +213,29 @@ static uintptr_t jl_object_id_(jl_value_t *tv, jl_value_t *v) return memhash32_seed(jl_string_data(v), jl_string_len(v), 0xedc3b677); #endif } - if (dt->mutabl) return inthash((uintptr_t)v); + if (dt->mutabl) + return inthash((uintptr_t)v); size_t sz = jl_datatype_size(tv); uintptr_t h = jl_object_id(tv); - if (sz == 0) return ~h; - size_t nf = jl_datatype_nfields(dt); - if (nf == 0) { + if (sz == 0) + return ~h; + size_t f, nf = jl_datatype_nfields(dt); + if (nf == 0) return bits_hash(jl_data_ptr(v), sz) ^ h; - } - for (size_t f=0; f < nf; f++) { + for (f = 0; f < nf; f++) { size_t offs = jl_field_offset(dt, f); char *vo = (char*)jl_data_ptr(v) + offs; uintptr_t u; if (jl_field_isptr(dt, f)) { jl_value_t *f = *(jl_value_t**)vo; - u = f==NULL ? 0 : jl_object_id(f); + u = (f == NULL) ? 0 : jl_object_id(f); } else { jl_datatype_t *fieldtype = (jl_datatype_t*)jl_field_type(dt, f); + if (jl_is_uniontype(fieldtype)) { + uint8_t sel = ((uint8_t*)vo)[jl_field_size(dt, f) - 1]; + fieldtype = (jl_datatype_t*)jl_nth_union_component((jl_value_t*)fieldtype, sel); + } assert(jl_is_datatype(fieldtype) && !fieldtype->abstract && !fieldtype->mutabl); if (fieldtype->layout->haspadding) u = jl_object_id_((jl_value_t*)fieldtype, (jl_value_t*)vo); @@ -244,7 +263,7 @@ JL_CALLABLE(jl_f_is) JL_NARGS(===, 2, 2); if (args[0] == args[1]) return jl_true; - return jl_egal(args[0],args[1]) ? jl_true : jl_false; + return jl_egal(args[0], args[1]) ? jl_true : jl_false; } JL_CALLABLE(jl_f_typeof) diff --git a/src/ccall.cpp b/src/ccall.cpp index 118e4ae3b2b84..d6dc4121114a5 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -182,7 +182,7 @@ static Value *runtime_sym_lookup( PHINode *p = irbuilder.CreatePHI(T_pvoidfunc, 2); p->addIncoming(llvmf_orig, enter_bb); p->addIncoming(llvmf, dlsym_lookup); - return irbuilder.CreatePointerCast(p, funcptype); + return irbuilder.CreateBitCast(p, funcptype); } static Value *runtime_sym_lookup( @@ -534,7 +534,7 @@ static Value *julia_to_address( { assert(jl_is_datatype(jlto) && julia_struct_has_layout((jl_datatype_t*)jlto, jlto_env)); - if (!jl_is_cpointer_type(jlto) || !to->isPointerTy()) { + if (!jl_is_cpointer_type(jlto) || to != T_size) { emit_error(ctx, "ccall: & on argument was not matched by Ptr{T} argument type"); return UndefValue::get(to); } @@ -547,19 +547,18 @@ static Value *julia_to_address( ety = jl_tparam0(jlto); typeassert_input(ctx, jvinfo, ety, jlto_env, argn, true); } - assert(to->isPointerTy()); if (jvinfo.isboxed) { if (!jl_is_abstracttype(ety)) { if (jl_is_mutable_datatype(ety)) { // no copy, just reference the data field - return data_pointer(ctx, jvinfo, to); + return ctx.builder.CreateBitCast(emit_pointer_from_objref(ctx, data_pointer(ctx, jvinfo)), to); } - else if (jl_is_immutable_datatype(ety) && jlto != (jl_value_t*)jl_voidpointer_type) { + else if (jl_is_immutable_datatype(ety) && jlto != (jl_value_t*)jl_voidpointer_type) { // anything declared `struct`, except Ptr{Void} // yes copy Value *nbytes; AllocaInst *ai; - if (jl_is_leaf_type(ety) || jl_is_primitivetype(ety)) { + if (((jl_datatype_t*)ety)->layout) { int nb = jl_datatype_size(ety); nbytes = ConstantInt::get(T_int32, nb); ai = emit_static_alloca(ctx, T_int8, nb); @@ -571,7 +570,7 @@ static Value *julia_to_address( } ai->setAlignment(16); ctx.builder.CreateMemCpy(ai, data_pointer(ctx, jvinfo, T_pint8), nbytes, sizeof(void*)); // minimum gc-alignment in julia is pointer size - return emit_bitcast(ctx, ai, to); + return ctx.builder.CreatePtrToInt(ai, to); } } // emit maybe copy @@ -583,14 +582,14 @@ static Value *julia_to_address( Value *ismutable = emit_datatype_mutabl(ctx, jvt); ctx.builder.CreateCondBr(ismutable, mutableBB, immutableBB); ctx.builder.SetInsertPoint(mutableBB); - Value *p1 = data_pointer(ctx, jvinfo, to); + Value *p1 = ctx.builder.CreateBitCast(emit_pointer_from_objref(ctx, data_pointer(ctx, jvinfo)), to); ctx.builder.CreateBr(afterBB); ctx.builder.SetInsertPoint(immutableBB); Value *nbytes = emit_datatype_size(ctx, jvt); AllocaInst *ai = ctx.builder.CreateAlloca(T_int8, nbytes); ai->setAlignment(16); ctx.builder.CreateMemCpy(ai, data_pointer(ctx, jvinfo, T_pint8), nbytes, sizeof(void*)); // minimum gc-alignment in julia is pointer size - Value *p2 = emit_bitcast(ctx, ai, to); + Value *p2 = ctx.builder.CreatePtrToInt(ai, to); ctx.builder.CreateBr(afterBB); ctx.builder.SetInsertPoint(afterBB); PHINode *p = ctx.builder.CreatePHI(to, 2); @@ -612,9 +611,7 @@ static Value *julia_to_address( (uint64_t)jl_datatype_size(ety), (uint64_t)jl_datatype_align(ety)); } - if (slot->getType() != to) - slot = emit_bitcast(ctx, slot, to); - return slot; + return ctx.builder.CreatePtrToInt(slot, to); } @@ -781,22 +778,21 @@ static jl_cgval_t emit_cglobal(jl_codectx_t &ctx, jl_value_t **args, size_t narg rt = (jl_value_t*)jl_voidpointer_type; } Type *lrt = julia_type_to_llvm(rt); - if (lrt == NULL) - lrt = T_pint8; interpret_symbol_arg(ctx, sym, args[1], "cglobal", false); if (sym.jl_ptr != NULL) { - res = ctx.builder.CreateIntToPtr(sym.jl_ptr, lrt); + res = ctx.builder.CreateBitCast(sym.jl_ptr, lrt); } else if (sym.fptr != NULL) { - res = literal_static_pointer_val(ctx, (void*)(uintptr_t)sym.fptr, lrt); + res = ConstantInt::get(lrt, (uint64_t)sym.fptr); if (imaging_mode) jl_printf(JL_STDERR,"WARNING: literal address used in cglobal for %s; code cannot be statically compiled\n", sym.f_name); } else { if (imaging_mode) { - res = runtime_sym_lookup(ctx, (PointerType*)lrt, sym.f_lib, sym.f_name, ctx.f); + res = runtime_sym_lookup(ctx, cast(T_pint8), sym.f_lib, sym.f_name, ctx.f); + res = ctx.builder.CreatePtrToInt(res, lrt); } else { void *symaddr = jl_dlsym_e(jl_get_library(sym.f_lib), sym.f_name); @@ -815,7 +811,7 @@ static jl_cgval_t emit_cglobal(jl_codectx_t &ctx, jl_value_t **args, size_t narg } // since we aren't saving this code, there's no sense in // putting anything complicated here: just JIT the address of the cglobal - res = literal_static_pointer_val(ctx, symaddr, lrt); + res = ConstantInt::get(lrt, (uint64_t)symaddr); } } @@ -1583,25 +1579,24 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) if (jl_is_long(argi_root)) continue; jl_cgval_t arg_root = emit_expr(ctx, argi_root); - Value *gcuse = arg_root.gcroot ? ctx.builder.CreateLoad(arg_root.gcroot) : arg_root.V; - if (gcuse) { - gc_uses.push_back(gcuse); + if (arg_root.Vboxed || arg_root.V) { + gc_uses.push_back(arg_root.Vboxed ? arg_root.Vboxed : arg_root.V); } } // some special functions if (is_libjulia_func(jl_array_ptr)) { - assert(lrt->isPointerTy()); + assert(lrt == T_size); assert(!isVa && !llvmcall && nargt == 1); assert(!addressOf.at(0)); const jl_cgval_t &ary = argv[0]; jl_value_t *aryex = ccallarg(0); JL_GC_POP(); - return mark_or_box_ccall_result(ctx, emit_bitcast(ctx, emit_arrayptr(ctx, ary, aryex), lrt), + return mark_or_box_ccall_result(ctx, ctx.builder.CreatePtrToInt(emit_arrayptr(ctx, ary, aryex), lrt), retboxed, rt, unionall, static_rt); } else if (is_libjulia_func(jl_value_ptr)) { - assert(lrt->isPointerTy()); + assert(retboxed ? lrt == T_prjlvalue : lrt == T_size); assert(!isVa && !llvmcall && nargt == 1); jl_value_t *tti = jl_svecref(at, 0); Value *ary; @@ -1613,7 +1608,7 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) } else if (jl_is_abstract_ref_type(tti)) { tti = (jl_value_t*)jl_voidpointer_type; - largty = T_pint8; + largty = T_size; isboxed = false; } else { @@ -1629,13 +1624,16 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) if (!retboxed) { return mark_or_box_ccall_result( ctx, - emit_bitcast(ctx, emit_pointer_from_objref(ctx, - emit_bitcast(ctx, ary, T_prjlvalue)), lrt), + emit_pointer_from_objref(ctx, + emit_bitcast(ctx, ary, T_prjlvalue)), retboxed, rt, unionall, static_rt); - } else { + } + else { return mark_or_box_ccall_result( ctx, - maybe_decay_untracked(emit_bitcast(ctx, ary, lrt)), + ctx.builder.CreateAddrSpaceCast( + ctx.builder.CreateIntToPtr(ary, T_pjlvalue), + T_prjlvalue), // TODO: this addrspace cast is invalid (implies that the value is rooted elsewhere) retboxed, rt, unionall, static_rt); } } @@ -1695,11 +1693,11 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) return ghostValue(jl_void_type); } else if (_is_libjulia_func((uintptr_t)ptls_getter, "jl_get_ptls_states")) { - assert(lrt == T_pint8); + assert(lrt == T_size); assert(!isVa && !llvmcall && nargt == 0); JL_GC_POP(); return mark_or_box_ccall_result(ctx, - emit_bitcast(ctx, ctx.ptlsStates, lrt), + ctx.builder.CreatePtrToInt(ctx.ptlsStates, lrt), retboxed, rt, unionall, static_rt); } else if (is_libjulia_func(jl_threadid)) { @@ -1772,6 +1770,7 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) } else if (is_libjulia_func(jl_function_ptr)) { assert(!isVa && !llvmcall && nargt == 3); + assert(lrt == T_size); jl_value_t *f = argv[0].constant; jl_value_t *frt = argv[1].constant; if (!frt) { @@ -1798,11 +1797,10 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) llvmf = NULL; } if (llvmf) { - llvmf = prepare_call(llvmf); JL_GC_POP(); JL_GC_POP(); - return mark_or_box_ccall_result(ctx, emit_bitcast(ctx, llvmf, lrt), - retboxed, rt, unionall, static_rt); + Value *fptr = ctx.builder.CreatePtrToInt(prepare_call(llvmf), lrt); + return mark_or_box_ccall_result(ctx, fptr, retboxed, rt, unionall, static_rt); } } JL_GC_POP(); @@ -1834,10 +1832,10 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) } } else if (is_libjulia_func(jl_string_ptr)) { - assert(lrt == T_pint8); + assert(lrt == T_size); assert(!isVa && !llvmcall && nargt == 1 && !addressOf.at(0)); - auto obj = emit_pointer_from_objref(ctx, boxed(ctx, argv[0])); - auto strp = ctx.builder.CreateConstGEP1_32(emit_bitcast(ctx, obj, T_pint8), sizeof(void*)); + Value *obj = emit_pointer_from_objref(ctx, boxed(ctx, argv[0])); + Value *strp = ctx.builder.CreateAdd(obj, ConstantInt::get(T_size, sizeof(void*))); JL_GC_POP(); return mark_or_box_ccall_result(ctx, strp, retboxed, rt, unionall, static_rt); } @@ -1940,9 +1938,6 @@ jl_cgval_t function_sig_t::emit_a_ccall( if (isa(v)) { return jl_cgval_t(); } - // A bit of a hack, but we're trying to get rid of this feature - // anyway. - v = emit_bitcast(ctx, emit_pointer_from_objref(ctx, v), pargty); assert((!toboxed && !byRef) || isa(v)); } diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 6d350e9ab94a9..73b54e3ffb2c1 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -170,15 +170,8 @@ static DIType *julia_type_to_di(jl_value_t *jt, DIBuilder *dbuilder, bool isboxe return jl_pvalue_dillvmt; jl_datatype_t *jdt = (jl_datatype_t*)jt; // always return the boxed representation for types with hidden content - if (jl_is_abstracttype(jt) || jl_is_array_type(jt) || - jt == (jl_value_t*)jl_sym_type || jt == (jl_value_t*)jl_module_type || - jt == (jl_value_t*)jl_simplevector_type || jt == (jl_value_t*)jl_datatype_type || - jt == (jl_value_t*)jl_method_instance_type) - return jl_pvalue_dillvmt; - if (jdt->ditype != NULL) { - DIType* t = (DIType*)jdt->ditype; - return t; - } + if (jdt->ditype != NULL) + return (DIType*)jdt->ditype; if (jl_is_primitivetype(jt)) { uint64_t SizeInBits = jl_datatype_nbits(jdt); #if JL_LLVM_VERSION >= 40000 @@ -198,11 +191,11 @@ static DIType *julia_type_to_di(jl_value_t *jt, DIBuilder *dbuilder, bool isboxe return t; #endif } - if (jl_is_structtype(jt) && jdt->layout) { + if (jl_is_structtype(jt) && jdt->uid && jdt->layout && !jl_is_layout_opaque(jdt->layout)) { size_t ntypes = jl_datatype_nfields(jdt); const char *tname = jl_symbol_name(jdt->name->name); std::stringstream unique_name; - unique_name << tname << "_" << globalUnique++; + unique_name << jdt->uid; llvm::DICompositeType *ct = dbuilder->createStructType( NULL, // Scope tname, // Name @@ -219,8 +212,15 @@ static DIType *julia_type_to_di(jl_value_t *jt, DIBuilder *dbuilder, bool isboxe ); jdt->ditype = ct; std::vector Elements; - for (unsigned i = 0; i < ntypes; i++) - Elements.push_back(julia_type_to_di(jl_svecref(jdt->types, i), dbuilder, false)); + for (unsigned i = 0; i < ntypes; i++) { + jl_value_t *el = jl_svecref(jdt->types, i); + DIType *di; + if (jl_field_isptr(jdt, i)) + di = jl_pvalue_dillvmt; + else + di = julia_type_to_di(el, dbuilder, false); + Elements.push_back(di); + } dbuilder->replaceArrays(ct, dbuilder->getOrCreateArray(ArrayRef(Elements))); return ct; } @@ -233,7 +233,7 @@ static Value *emit_pointer_from_objref(jl_codectx_t &ctx, Value *V) { unsigned AS = cast(V->getType())->getAddressSpace(); if (AS != AddressSpace::Tracked && AS != AddressSpace::Derived) - return ctx.builder.CreateBitCast(V, T_pjlvalue); + return ctx.builder.CreatePtrToInt(V, T_size); V = ctx.builder.CreateBitCast(decay_derived(V), PointerType::get(T_jlvalue, AddressSpace::Derived)); CallInst *Call = ctx.builder.CreateCall(prepare_call(pointer_from_objref_func), V); @@ -427,18 +427,14 @@ static Type *bitstype_to_llvm(jl_value_t *bt) assert(jl_is_primitivetype(bt)); if (bt == (jl_value_t*)jl_bool_type) return T_int8; - if (bt == (jl_value_t*)jl_long_type) - return T_size; + if (bt == (jl_value_t*)jl_int32_type) + return T_int32; + if (bt == (jl_value_t*)jl_int64_type) + return T_int64; if (bt == (jl_value_t*)jl_float32_type) return T_float32; if (bt == (jl_value_t*)jl_float64_type) return T_float64; - if (jl_is_cpointer_type(bt)) { - Type *lt = julia_type_to_llvm(jl_tparam0(bt)); - if (lt == T_void) - return T_pint8; - return PointerType::get(lt, 0); - } int nb = jl_datatype_size(bt); return Type::getIntNTy(jl_LLVMContext, nb * 8); } @@ -461,6 +457,14 @@ static bool julia_struct_has_layout(jl_datatype_t *dt, jl_unionall_t *ua) return true; } +static unsigned jl_field_align(jl_datatype_t *dt, size_t i) +{ + unsigned al = jl_field_offset(dt, i); + al |= 16; + al &= -al; + return std::min(al, jl_datatype_align(dt)); +} + static Type *julia_struct_to_llvm(jl_value_t *jt, jl_unionall_t *ua, bool *isboxed) { // this function converts a Julia Type into the equivalent LLVM struct @@ -473,93 +477,117 @@ static Type *julia_struct_to_llvm(jl_value_t *jt, jl_unionall_t *ua, bool *isbox return bitstype_to_llvm(jt); bool isTuple = jl_is_tuple_type(jt); jl_datatype_t *jst = (jl_datatype_t*)jt; + if (jst->struct_decl != NULL) + return (Type*)jst->struct_decl; if (jl_is_structtype(jt) && !(jst->layout && jl_is_layout_opaque(jst->layout))) { - if (jst->struct_decl == NULL) { - size_t i, ntypes = jl_svec_len(jst->types); - if (ntypes == 0 || (jst->layout && jl_datatype_nbits(jst) == 0)) - return T_void; - if (!julia_struct_has_layout(jst, ua)) - return NULL; - StructType *structdecl; - if (!isTuple) { - structdecl = StructType::create(jl_LLVMContext, jl_symbol_name(jst->name->name)); - jst->struct_decl = structdecl; - } - std::vector latypes(0); - bool isarray = true; - bool isvector = true; - jl_value_t *jlasttype = NULL; - Type *lasttype = NULL; - bool allghost = true; - for (i = 0; i < ntypes; i++) { - jl_value_t *ty = jl_svecref(jst->types, i); - if (jlasttype != NULL && ty != jlasttype) - isvector = false; - jlasttype = ty; - bool isptr; - if (jst->layout) - isptr = jl_field_isptr(jst, i); - else // compute what jl_compute_field_offsets would say - isptr = jl_isbits(ty) && jl_is_leaf_type(ty) && ((jl_datatype_t*)ty)->layout; - Type *lty; - if (isptr) - lty = T_pjlvalue; - else if (ty == (jl_value_t*)jl_bool_type) - lty = T_int8; - else - lty = julia_type_to_llvm(ty); - if (lasttype != NULL && lasttype != lty) - isarray = false; - lasttype = lty; - if (type_is_ghost(lty)) - lty = NoopType; - else - allghost = false; - latypes.push_back(lty); - } - if (allghost) { - assert(jst->layout == NULL); // otherwise should have been caught above - jst->struct_decl = T_void; + size_t i, ntypes = jl_svec_len(jst->types); + if (ntypes == 0 || (jst->layout && jl_datatype_nbits(jst) == 0)) + return T_void; + if (!julia_struct_has_layout(jst, ua)) + return NULL; + std::vector latypes(ntypes); + bool isarray = true; + bool isvector = true; + jl_value_t *jlasttype = NULL; + Type *lasttype = NULL; + bool allghost = true; + for (i = 0; i < ntypes; i++) { + jl_value_t *ty = jl_svecref(jst->types, i); + if (jlasttype != NULL && ty != jlasttype) + isvector = false; + jlasttype = ty; + bool isptr; + size_t fsz = 0, al = 0; + if (jst->layout) { + isptr = jl_field_isptr(jst, i); + fsz = jl_field_size(jst, i); + al = jl_field_align(jst, i); } - else if (!isTuple) { - if (jl_is_vecelement_type(jt)) - // VecElement type is unwrapped in LLVM - jst->struct_decl = latypes[0]; - else - structdecl->setBody(latypes); + else { // compute what jl_compute_field_offsets would say + isptr = !jl_islayout_inline(ty, &fsz, &al); + if (!isptr && jl_is_uniontype(jst)) + fsz += 1; } - else { - if (isarray && lasttype != T_int1 && !type_is_ghost(lasttype)) { - if (isvector && jl_special_vector_alignment(ntypes, jlasttype) != 0) - jst->struct_decl = VectorType::get(lasttype, ntypes); - else - jst->struct_decl = ArrayType::get(lasttype, ntypes); - } - else { - jst->struct_decl = StructType::get(jl_LLVMContext, ArrayRef(&latypes[0], ntypes)); - } + Type *lty; + if (isptr) + lty = T_pjlvalue; + else if (ty == (jl_value_t*)jl_bool_type) + lty = T_int8; + else if (jl_is_uniontype(ty)) { + // pick an Integer type size such that alignment will be correct + // and always end with an Int8 (selector byte) + lty = ArrayType::get(IntegerType::get(jl_LLVMContext, 8 * al), (fsz - 1) / al); + std::vector Elements(2); + Elements[0] = lty; + Elements[1] = T_int8; + unsigned remainder = (fsz - 1) % al; + while (remainder--) + Elements.push_back(T_int8); + lty = StructType::get(jl_LLVMContext, Elements); } + else + lty = julia_type_to_llvm(ty); + if (lasttype != NULL && lasttype != lty) + isarray = false; + lasttype = lty; + if (type_is_ghost(lty)) + lty = NoopType; + else + allghost = false; + latypes[i] = lty; + } + Type *decl; + if (allghost) { + assert(jst->layout == NULL); // otherwise should have been caught above + decl = T_void; + } + else if (jl_is_vecelement_type(jt)) { + // VecElement type is unwrapped in LLVM + decl = latypes[0]; + } + else if (isTuple && isarray && lasttype != T_int1 && !type_is_ghost(lasttype)) { + if (isvector && jl_special_vector_alignment(ntypes, jlasttype) != 0) + decl = VectorType::get(lasttype, ntypes); + else + decl = ArrayType::get(lasttype, ntypes); + } + else { + decl = StructType::get(jl_LLVMContext, latypes); + } + jst->struct_decl = decl; #ifndef JL_NDEBUG - // If LLVM and Julia disagree about alignment, much trouble ensues, so check it! - if (jst->layout) { - const DataLayout &DL = + // If LLVM and Julia disagree about alignment, much trouble ensues, so check it! + if (jst->layout) { + const DataLayout &DL = #if JL_LLVM_VERSION >= 40000 - jl_data_layout; + jl_data_layout; #else - jl_ExecutionEngine->getDataLayout(); -#endif - unsigned llvm_alignment = DL.getABITypeAlignment((Type*)jst->struct_decl); - unsigned julia_alignment = jl_datatype_align(jst); - // Check that the alignment adheres to the heap alignment. - assert(julia_alignment <= JL_HEAP_ALIGNMENT); - // TODO: Fix alignment calculation in LLVM, as well as in the GC and the struct declaration - if (llvm_alignment <= JL_HEAP_ALIGNMENT) - assert(julia_alignment == llvm_alignment); - } + jl_ExecutionEngine->getDataLayout(); #endif + unsigned llvm_alignment = DL.getABITypeAlignment((Type*)jst->struct_decl); + unsigned julia_alignment = jl_datatype_align(jst); + // Check that the alignment adheres to the heap alignment. + assert(julia_alignment <= JL_HEAP_ALIGNMENT); + // TODO: Fix alignment calculation in LLVM, as well as in the GC and the struct declaration + if (llvm_alignment <= JL_HEAP_ALIGNMENT) + assert(julia_alignment == llvm_alignment); } - return (Type*)jst->struct_decl; - } +#endif + return decl; + } + // TODO: enable this (with tests): + // if (jl_is_uniontype(ty)) { + // // pick an Integer type size such that alignment will be correct + // // and always end with an Int8 (selector byte) + // lty = ArrayType::get(IntegerType::get(jl_LLVMContext, 8 * al), (fsz - 1) / al); + // std::vector Elements(2); + // Elements[0] = lty; + // Elements[1] = T_int8; + // unsigned remainder = (fsz - 1) % al; + // while (remainder--) + // Elements.push_back(T_int8); + // lty = StructType::get(jl_LLVMContext, makeArrayRef(Elements)); + // } if (isboxed) *isboxed = true; return T_pjlvalue; } @@ -567,7 +595,7 @@ static Type *julia_struct_to_llvm(jl_value_t *jt, jl_unionall_t *ua, bool *isbox static bool is_datatype_all_pointers(jl_datatype_t *dt) { size_t i, l = jl_datatype_nfields(dt); - for(i=0; i < l; i++) { + for (i = 0; i < l; i++) { if (!jl_field_isptr(dt, i)) { return false; } @@ -586,9 +614,9 @@ static bool is_tupletype_homogeneous(jl_svec_t *t, bool allow_va = false) return true; return false; } - for(i=1; i < l; i++) { - if (allow_va && i == l - 1 && jl_is_vararg_type(jl_svecref(t,i))) { - if (t0 != jl_unwrap_vararg(jl_svecref(t,i))) + for (i = 1; i < l; i++) { + if (allow_va && i == l - 1 && jl_is_vararg_type(jl_svecref(t, i))) { + if (t0 != jl_unwrap_vararg(jl_svecref(t, i))) return false; continue; } @@ -728,7 +756,7 @@ static jl_cgval_t emit_typeof(jl_codectx_t &ctx, const jl_cgval_t &p) else { // See note above in emit_typeof(Value*), we can't tell the system // about this until we've cleared the GC bits. - pdatatype = emit_bitcast(ctx, emit_typeptr_addr(ctx, ctx.builder.CreateLoad(p.gcroot)), T_ppjlvalue); + pdatatype = emit_bitcast(ctx, emit_typeptr_addr(ctx, p.Vboxed), T_ppjlvalue); } counter = 0; for_each_uniontype_small( @@ -1028,7 +1056,7 @@ static std::pair emit_isa(jl_codectx_t &ctx, const jl_cgval_t &x, Value *xtindex = ctx.builder.CreateAnd(x.TIndex, ConstantInt::get(T_int8, 0x7f)); return std::make_pair(ctx.builder.CreateICmpEQ(xtindex, ConstantInt::get(T_int8, tindex)), false); } - else { + else if (x.Vboxed) { // test for (x.TIndex == 0x80 && typeof(x.V) == type) Value *isboxed = ctx.builder.CreateICmpEQ(x.TIndex, ConstantInt::get(T_int8, 0x80)); BasicBlock *currBB = ctx.builder.GetInsertBlock(); @@ -1036,7 +1064,7 @@ static std::pair emit_isa(jl_codectx_t &ctx, const jl_cgval_t &x, BasicBlock *postBB = BasicBlock::Create(jl_LLVMContext, "post_isa", ctx.f); ctx.builder.CreateCondBr(isboxed, isaBB, postBB); ctx.builder.SetInsertPoint(isaBB); - Value *istype_boxed = ctx.builder.CreateICmpEQ(emit_typeof(ctx, x.V), + Value *istype_boxed = ctx.builder.CreateICmpEQ(emit_typeof(ctx, x.Vboxed), maybe_decay_untracked(literal_pointer_val(ctx, type))); ctx.builder.CreateBr(postBB); ctx.builder.SetInsertPoint(postBB); @@ -1044,6 +1072,9 @@ static std::pair emit_isa(jl_codectx_t &ctx, const jl_cgval_t &x, istype->addIncoming(ConstantInt::get(T_int1, 0), currBB); istype->addIncoming(istype_boxed, isaBB); return std::make_pair(istype, false); + } else { + // handle the case where we know that `x` is unboxed (but of unknown type), but that leaf type `type` cannot be unboxed + return std::make_pair(ConstantInt::get(T_int1, 0), false); } } return std::make_pair(ctx.builder.CreateICmpEQ(emit_typeof_boxed(ctx, x), @@ -1170,9 +1201,10 @@ static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, j return ghostValue(jltype); if (isboxed) elty = T_prjlvalue; - Value *data = ptr; - if (ptr->getType()->getContainedType(0) != elty) - data = emit_bitcast(ctx, ptr, PointerType::get(elty, 0)); + Type *ptrty = PointerType::get(elty, ptr->getType()->getPointerAddressSpace()); + Value *data; + if (ptr->getType() != ptrty) + data = emit_bitcast(ctx, ptr, ptrty); else data = ptr; if (idx_0based) @@ -1220,14 +1252,16 @@ static void typed_store(jl_codectx_t &ctx, emit_write_barrier(ctx, parent, r); } Value *data; - if (ptr->getType()->getContainedType(0) != elty) { + Type *ptrty = PointerType::get(elty, ptr->getType()->getPointerAddressSpace()); + if (ptr->getType() != ptrty) { if (isboxed) { data = emit_bitcast(ctx, ptr, T_pprjlvalue); } else { - data = emit_bitcast(ctx, ptr, PointerType::get(elty, cast(ptr->getType())->getAddressSpace())); + data = emit_bitcast(ctx, ptr, ptrty); } - } else + } else { data = ptr; + } Instruction *store = ctx.builder.CreateAlignedStore(r, ctx.builder.CreateGEP(data, idx_0based), isboxed ? alignment : julia_alignment(r, jltype, alignment)); if (tbaa) @@ -1299,7 +1333,6 @@ static bool emit_getfield_unknownidx(jl_codectx_t &ctx, Type *fty = julia_type_to_llvm(jt); Value *addr = ctx.builder.CreateGEP(emit_bitcast(ctx, decay_derived(ptr), PointerType::get(fty,0)), idx); *ret = mark_julia_slot(addr, jt, NULL, strct.tbaa); - ret->gcroot = strct.gcroot; ret->isimmutable = strct.isimmutable; return true; } @@ -1376,9 +1409,7 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st addr = ctx.builder.CreateStructGEP(lt, ptr, idx); } } - int align = jl_field_offset(jt, idx); - align |= 16; - align &= -align; + int align = jl_field_align(jt, idx); if (jl_field_isptr(jt, idx)) { bool maybe_null = idx >= (unsigned)jt->ninitialized; Instruction *Load = maybe_mark_load_dereferenceable( @@ -1395,7 +1426,6 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st Value *ptindex = ctx.builder.CreateGEP(T_int8, emit_bitcast(ctx, addr, T_pint8), ConstantInt::get(T_size, fsz - 1)); Value *tindex = ctx.builder.CreateNUWAdd(ConstantInt::get(T_int8, 1), ctx.builder.CreateLoad(ptindex)); bool isimmutable = strct.isimmutable; - Value *gcroot = strct.gcroot; if (jt->mutabl) { // move value to an immutable stack slot Type *AT = ArrayType::get(IntegerType::get(jl_LLVMContext, 8 * align), (fsz + align - 2) / align); @@ -1406,18 +1436,15 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st ctx.builder.CreateMemCpy(lv, addr, nbytes, align); addr = lv; isimmutable = true; - gcroot = NULL; } jl_cgval_t fieldval = mark_julia_slot(addr, jfty, tindex, strct.tbaa); fieldval.isimmutable = isimmutable; - fieldval.gcroot = gcroot; return fieldval; } else if (!jt->mutabl) { // just compute the pointer and let user load it when necessary jl_cgval_t fieldval = mark_julia_slot(addr, jfty, NULL, strct.tbaa); fieldval.isimmutable = strct.isimmutable; - fieldval.gcroot = strct.gcroot; return fieldval; } return typed_load(ctx, addr, ConstantInt::get(T_size, 0), jfty, strct.tbaa, true, align); @@ -1897,6 +1924,8 @@ static Value *compute_box_tindex(jl_codectx_t &ctx, Value *datatype, jl_value_t // get the runtime tindex value static Value *compute_tindex_unboxed(jl_codectx_t &ctx, const jl_cgval_t &val, jl_value_t *typ) { + if (val.typ == jl_bottom_type) + return UndefValue::get(T_int8); if (val.constant) return ConstantInt::get(T_int8, get_box_tindex((jl_datatype_t*)jl_typeof(val.constant), typ)); if (val.isboxed) @@ -1905,6 +1934,13 @@ static Value *compute_tindex_unboxed(jl_codectx_t &ctx, const jl_cgval_t &val, j return ctx.builder.CreateAnd(val.TIndex, ConstantInt::get(T_int8, 0x7f)); } +/* + * Box unboxed values in a union. Optionally, skip certain unboxed values, + * returning `V_null` in one of the skipped cases. If `skip` is not empty, + * skip[0] (corresponding to unknown boxed) must always be set. In that + * case, the calling code must separately deal with the case where + * `vinfo` is already an unkown boxed union (union tag 0x80). + */ static Value *box_union(jl_codectx_t &ctx, const jl_cgval_t &vinfo, const SmallBitVector &skip) { // given vinfo::Union{T, S}, emit IR of the form: @@ -1955,13 +1991,12 @@ static Value *box_union(jl_codectx_t &ctx, const jl_cgval_t &vinfo, const SmallB vinfo.typ, counter); ctx.builder.SetInsertPoint(defaultBB); - if (skip.size() > 0 && skip[0]) { - // skip[0] specifies where to return NULL or the original pointer - // if the value was not handled above + if (skip.size() > 0) { + assert(skip[0]); box_merge->addIncoming(maybe_decay_untracked(V_null), defaultBB); ctx.builder.CreateBr(postBB); } - else if ((vinfo.V == NULL || isa(vinfo.V)) && !vinfo.gcroot) { + else if (!vinfo.Vboxed) { Function *trap_func = Intrinsic::getDeclaration( ctx.f->getParent(), Intrinsic::trap); @@ -1969,9 +2004,7 @@ static Value *box_union(jl_codectx_t &ctx, const jl_cgval_t &vinfo, const SmallB ctx.builder.CreateUnreachable(); } else { - // We're guaranteed here that Load(.gcroot) == .V, because we have determined - // that this union is a boxed value, rather than an interior pointer of some sort - box_merge->addIncoming(ctx.builder.CreateLoad(vinfo.gcroot), defaultBB); + box_merge->addIncoming(vinfo.Vboxed, defaultBB); ctx.builder.CreateBr(postBB); } ctx.builder.SetInsertPoint(postBB); @@ -1990,10 +2023,8 @@ static Value *boxed(jl_codectx_t &ctx, const jl_cgval_t &vinfo) if (vinfo.constant) return maybe_decay_untracked(literal_pointer_val(ctx, vinfo.constant)); if (vinfo.isboxed) { - assert(vinfo.V && "Missing value for box."); - // We're guaranteed here that Load(.gcroot) == .V, because we have determined - // that this value is a box, so if it has a gcroot, that's where the value is. - return vinfo.gcroot ? ctx.builder.CreateLoad(vinfo.gcroot) : vinfo.V; + assert(vinfo.V == vinfo.Vboxed); + return vinfo.V; } Value *box; @@ -2184,28 +2215,26 @@ static void emit_setfield(jl_codectx_t &ctx, if (wb && strct.isboxed) emit_checked_write_barrier(ctx, boxed(ctx, strct), r); } - else { - if (jl_is_uniontype(jfty)) { - int fsz = jl_field_size(sty, idx0); - // compute tindex from rhs - jl_cgval_t rhs_union = convert_julia_type(ctx, rhs, jfty); - Value *tindex = compute_tindex_unboxed(ctx, rhs_union, jfty); - tindex = ctx.builder.CreateNUWSub(tindex, ConstantInt::get(T_int8, 1)); - Value *ptindex = ctx.builder.CreateGEP(T_int8, emit_bitcast(ctx, addr, T_pint8), ConstantInt::get(T_size, fsz - 1)); - ctx.builder.CreateStore(tindex, ptindex); - // copy data - if (!rhs.isghost) { - emit_unionmove(ctx, addr, rhs, NULL, false, NULL); - } - } - else { - int align = jl_field_offset(sty, idx0); - align |= 16; - align &= -align; - typed_store(ctx, addr, ConstantInt::get(T_size, 0), rhs, jfty, - strct.tbaa, data_pointer(ctx, strct, T_pjlvalue), align); + else if (jl_is_uniontype(jfty)) { + int fsz = jl_field_size(sty, idx0); + // compute tindex from rhs + jl_cgval_t rhs_union = convert_julia_type(ctx, rhs, jfty); + if (rhs_union.typ == jl_bottom_type) + return; + Value *tindex = compute_tindex_unboxed(ctx, rhs_union, jfty); + tindex = ctx.builder.CreateNUWSub(tindex, ConstantInt::get(T_int8, 1)); + Value *ptindex = ctx.builder.CreateGEP(T_int8, emit_bitcast(ctx, addr, T_pint8), ConstantInt::get(T_size, fsz - 1)); + ctx.builder.CreateStore(tindex, ptindex); + // copy data + if (!rhs.isghost) { + emit_unionmove(ctx, addr, rhs, NULL, false, NULL); } } + else { + int align = jl_field_align(sty, idx0); + typed_store(ctx, addr, ConstantInt::get(T_size, 0), rhs, jfty, + strct.tbaa, data_pointer(ctx, strct, T_pjlvalue), align); + } } else { // TODO: better error @@ -2223,50 +2252,81 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg if (nf > 0) { if (jl_isbits(sty)) { Type *lt = julia_type_to_llvm(ty); + unsigned na = (nargs - 1 < nf) ? (nargs - 1) : nf; + // whether we should perform the initialization with the struct as a IR value // or instead initialize the stack buffer with stores bool init_as_value = false; if (lt->isVectorTy() || - jl_is_vecelement_type(ty) || - type_is_ghost(lt)) // maybe also check the size ? + jl_is_vecelement_type(ty)) { // maybe also check the size ? init_as_value = true; + } - size_t na = nargs-1 < nf ? nargs-1 : nf; Value *strct; - if (init_as_value) - strct = UndefValue::get(lt == T_void ? NoopType : lt); + if (type_is_ghost(lt)) + strct = NULL; + else if (init_as_value) + strct = UndefValue::get(lt); else strct = emit_static_alloca(ctx, lt); - unsigned idx = 0; - for (size_t i = 0; i < na; i++) { + for (unsigned i = 0; i < na; i++) { jl_value_t *jtype = jl_svecref(sty->types, i); - Type *fty = julia_type_to_llvm(jtype); const jl_cgval_t &fval_info = argv[i + 1]; emit_typecheck(ctx, fval_info, jtype, "new"); - if (!type_is_ghost(fty)) { - Value *fval = NULL, *dest = NULL; - if (!init_as_value) { - // avoid unboxing the argument explicitly - // and use memcpy instead - dest = ctx.builder.CreateConstInBoundsGEP2_32(lt, strct, 0, i); - } + Type *fty; + if (type_is_ghost(lt)) + continue; + else if (jl_is_vecelement_type(ty)) + fty = lt; + else + fty = cast(lt)->getTypeAtIndex(i); + if (type_is_ghost(fty)) + continue; + Value *dest = NULL; + if (!init_as_value) { + // avoid unboxing the argument explicitly + // and use memcpy instead + dest = ctx.builder.CreateConstInBoundsGEP2_32(lt, strct, 0, i); + } + Value *fval = NULL; + if (jl_is_uniontype(jtype)) { + assert(!init_as_value && "unimplemented"); + StructType *lt_i = cast(fty); + // compute tindex from rhs + jl_cgval_t rhs_union = convert_julia_type(ctx, fval_info, jtype); + if (rhs_union.typ == jl_bottom_type) + return jl_cgval_t(); + Value *tindex = compute_tindex_unboxed(ctx, rhs_union, jtype); + tindex = ctx.builder.CreateNUWSub(tindex, ConstantInt::get(T_int8, 1)); + Value *ptindex = ctx.builder.CreateConstInBoundsGEP2_32( + lt_i, dest, 0, lt_i->getNumElements() - 1); + ctx.builder.CreateStore(tindex, ptindex); + if (!rhs_union.isghost) + emit_unionmove(ctx, dest, fval_info, NULL, false, NULL); + // If you wanted to implement init_as_value, + // would need to emit the union-move into temporary memory, + // then load it and combine with the tindex. + // But more efficient to just store it directly. + } + else { fval = emit_unbox(ctx, fty, fval_info, jtype, dest); - - if (init_as_value) { - if (lt->isVectorTy()) - strct = ctx.builder.CreateInsertElement(strct, fval, ConstantInt::get(T_int32, idx)); - else if (jl_is_vecelement_type(ty)) - strct = fval; // VecElement type comes unwrapped in LLVM. - else if (lt->isAggregateType()) - strct = ctx.builder.CreateInsertValue(strct, fval, ArrayRef(&idx, 1)); - else - assert(false); - } } - idx++; + if (init_as_value) { + assert(fval); + if (lt->isVectorTy()) + strct = ctx.builder.CreateInsertElement(strct, fval, ConstantInt::get(T_int32, i)); + else if (jl_is_vecelement_type(ty)) + strct = fval; // VecElement type comes unwrapped in LLVM. + else if (lt->isAggregateType()) + strct = ctx.builder.CreateInsertValue(strct, fval, ArrayRef(&i, 1)); + else + assert(false); + } } - if (init_as_value) + if (type_is_ghost(lt)) + return mark_julia_const(sty->instance); + else if (init_as_value) return mark_julia_type(ctx, strct, false, ty); else return mark_julia_slot(strct, ty, NULL, tbaa_stack); diff --git a/src/codegen.cpp b/src/codegen.cpp index 9aee554c1e84f..52a8e2583d1fc 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -391,9 +391,14 @@ static MDNode *best_tbaa(jl_value_t *jt) { // metadata tracking for a llvm Value* during codegen struct jl_cgval_t { Value *V; // may be of type T* or T, or set to NULL if ghost (or if the value has not been initialized yet, for a variable definition) + // For unions, we may need to keep a reference to the boxed part individually. + // If this is non-NULL, then, at runtime, we satisfy the invariant that (for the corresponding + // runtime values) if `(TIndex | 0x80) != 0`, then `Vboxed == V` (by value). + // For conenience, we also set this value of isboxed values, in which case + // it is equal (at compile time) to V. + Value *Vboxed; Value *TIndex; // if `V` is an unboxed (tagged) Union described by `typ`, this gives the DataType index (1-based, small int) as an i8 jl_value_t *constant; // constant value (rooted in linfo.def.roots) - Value *gcroot; // the gcroot associated with V (if it has one) jl_value_t *typ; // the original type of V, never NULL bool isboxed; // whether this value is a jl_value_t* allocated on the heap with the right type tag bool isghost; // whether this value is "ghost" @@ -411,23 +416,24 @@ struct jl_cgval_t { //} jl_cgval_t(Value *V, Value *gcroot, bool isboxed, jl_value_t *typ, Value *tindex) : // general constructor (with pointer type auto-detect) V(V), // V is allowed to be NULL in a jl_varinfo_t context, but not during codegen contexts + Vboxed(isboxed ? V : nullptr), TIndex(tindex), constant(NULL), - gcroot(gcroot), typ(typ), isboxed(isboxed), isghost(false), isimmutable(isboxed && jl_is_immutable_datatype(typ)), tbaa(isboxed ? best_tbaa(typ) : nullptr) { + assert(gcroot == nullptr); assert(!(isboxed && TIndex != NULL)); assert(TIndex == NULL || TIndex->getType() == T_int8); } jl_cgval_t(jl_value_t *typ) : // ghost value constructor V(NULL), + Vboxed(NULL), TIndex(NULL), constant(((jl_datatype_t*)typ)->instance), - gcroot(NULL), typ(typ), isboxed(false), isghost(true), @@ -439,9 +445,9 @@ struct jl_cgval_t { } jl_cgval_t(const jl_cgval_t &v, jl_value_t *typ, Value *tindex) : // copy constructor with new type V(v.V), + Vboxed(v.Vboxed), TIndex(tindex), constant(v.constant), - gcroot(v.gcroot), typ(typ), isboxed(v.isboxed), isghost(v.isghost), @@ -459,9 +465,9 @@ struct jl_cgval_t { } jl_cgval_t() : // undef / unreachable / default constructor V(UndefValue::get(T_void)), + Vboxed(NULL), TIndex(NULL), constant(NULL), - gcroot(NULL), typ(jl_bottom_type), isboxed(false), isghost(true), @@ -570,7 +576,6 @@ class jl_codectx_t { }; static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr); -static Value *emit_local_root(jl_codectx_t &ctx, jl_varinfo_t *vi = NULL); static Value *global_binding_pointer(jl_codectx_t &ctx, jl_module_t *m, jl_sym_t *s, jl_binding_t **pbnd, bool assign); static jl_cgval_t emit_checked_var(jl_codectx_t &ctx, Value *bp, jl_sym_t *name, bool isvol, MDNode *tbaa); @@ -714,8 +719,8 @@ static inline jl_cgval_t update_julia_type(jl_codectx_t &ctx, const jl_cgval_t & return v; // not worth trying to improve type info if (!isbits_spec(typ)) { // discovered that this union-split type must actually be isboxed - if (v.V) { - return jl_cgval_t(v.V, v.gcroot, true, typ, NULL); + if (v.Vboxed) { + return jl_cgval_t(v.Vboxed, nullptr, true, typ, NULL); } else { // type mismatch (there weren't any boxed values in the union) @@ -730,7 +735,7 @@ static inline jl_cgval_t update_julia_type(jl_codectx_t &ctx, const jl_cgval_t & return jl_cgval_t(v, typ, NULL); } -static jl_cgval_t convert_julia_type(jl_codectx_t &ctx, const jl_cgval_t &v, jl_value_t *typ, bool needsroot = true); +static jl_cgval_t convert_julia_type(jl_codectx_t &ctx, const jl_cgval_t &v, jl_value_t *typ); // --- allocating local variables --- @@ -811,8 +816,154 @@ static void jl_rethrow_with_add(const char *fmt, ...) jl_rethrow(); } +static jl_cgval_t convert_julia_type_union(jl_codectx_t &ctx, const jl_cgval_t &v, jl_value_t *typ) +{ + // previous value was a split union, compute new index, or box + Value *new_tindex = ConstantInt::get(T_int8, 0x80); + SmallBitVector skip_box(1, true); + Value *tindex = ctx.builder.CreateAnd(v.TIndex, ConstantInt::get(T_int8, 0x7f)); + if (jl_is_uniontype(typ)) { + // compute the TIndex mapping from v.typ -> typ + unsigned counter = 0; + for_each_uniontype_small( + // for each old union-split value + [&](unsigned idx, jl_datatype_t *jt) { + unsigned new_idx = get_box_tindex(jt, typ); + bool t; + if (new_idx) { + // found a matching element, + // match it against either the unboxed index + Value *cmp = ctx.builder.CreateICmpEQ(tindex, ConstantInt::get(T_int8, idx)); + new_tindex = ctx.builder.CreateSelect(cmp, ConstantInt::get(T_int8, new_idx), new_tindex); + t = true; + } + else if (!jl_subtype((jl_value_t*)jt, typ)) { + // new value doesn't need to be boxed + // since it isn't part of the new union + t = true; + } + else { + // will actually need to box this element + // since it appeared as a leaftype in the original type + // but not in the remark type + t = false; + } + skip_box.resize(idx + 1, t); + }, + v.typ, + counter); + } + + // some of the values are still unboxed + if (!isa(new_tindex)) { + Value *wasboxed = NULL; + // If the old value was boxed and unknown (type tag 0x80), + // it is possible that the tag was actually one of the types + // that are now explicitly represented. To find out, we need + // to compare typeof(v.Vboxed) (i.e. the type of the unknown + // value) against all the types that are now explicitly + // selected and select the appropriate one as our new tindex. + if (v.Vboxed) { + wasboxed = ctx.builder.CreateAnd(v.TIndex, ConstantInt::get(T_int8, 0x80)); + new_tindex = ctx.builder.CreateOr(wasboxed, new_tindex); + wasboxed = ctx.builder.CreateICmpNE(wasboxed, ConstantInt::get(T_int8, 0)); + + BasicBlock *currBB = ctx.builder.GetInsertBlock(); + + // We lazily create a BB for this, once we decide that we + // actually need it. + Value *union_box_dt = NULL; + BasicBlock *union_isaBB = NULL; + auto maybe_setup_union_isa = [&]() { + union_isaBB = BasicBlock::Create(jl_LLVMContext, "union_isa", ctx.f); + ctx.builder.SetInsertPoint(union_isaBB); + union_box_dt = emit_typeof(ctx, v.Vboxed); + }; + + // If we don't find a match. The type remains unknown + // (0x80). We could use `v.Tindex`, here, since we know + // it has to be 0x80, but it seems likely the backend + // will like the explicit constant better. + Value *union_box_tindex = ConstantInt::get(T_int8, 0x80); + unsigned counter = 0; + for_each_uniontype_small( + // for each new union-split value + [&](unsigned idx, jl_datatype_t *jt) { + unsigned old_idx = get_box_tindex(jt, v.typ); + if (old_idx == 0) { + // didn't handle this item before, select its new union index + maybe_setup_union_isa(); + Value *cmp = ctx.builder.CreateICmpEQ(maybe_decay_untracked(literal_pointer_val(ctx, (jl_value_t*)jt)), union_box_dt); + union_box_tindex = ctx.builder.CreateSelect(cmp, ConstantInt::get(T_int8, 0x80 | idx), union_box_tindex); + } + }, + typ, + counter); + if (union_box_dt) { + BasicBlock *postBB = BasicBlock::Create(jl_LLVMContext, "post_union_isa", ctx.f); + ctx.builder.CreateBr(postBB); + ctx.builder.SetInsertPoint(currBB); + Value *wasunknown = ctx.builder.CreateICmpEQ(v.TIndex, ConstantInt::get(T_int8, 0x80)); + ctx.builder.CreateCondBr(wasunknown, union_isaBB, postBB); + ctx.builder.SetInsertPoint(postBB); + PHINode *tindex_phi = ctx.builder.CreatePHI(T_int8, 2); + tindex_phi->addIncoming(new_tindex, currBB); + tindex_phi->addIncoming(union_box_tindex, union_isaBB); + new_tindex = tindex_phi; + } + } + if (!skip_box.all()) { + // some values weren't unboxed in the new union + // box them now (tindex above already selected 0x80 = box for them) + Value *boxv = box_union(ctx, v, skip_box); + if (v.Vboxed) { + // If the value is boxed both before and after, we don't need + // to touch it at all. Otherwise we're either transitioning + // unboxed->boxed, or leaving an unboxed value in place. + Value *isboxed = ctx.builder.CreateICmpNE( + ctx.builder.CreateAnd(new_tindex, ConstantInt::get(T_int8, 0x80)), + ConstantInt::get(T_int8, 0)); + boxv = ctx.builder.CreateSelect( + ctx.builder.CreateAnd(wasboxed, isboxed), v.Vboxed, boxv); + } + if (v.V == NULL) { + // v.V might be NULL if it was all ghost objects before + return jl_cgval_t(boxv, NULL, false, typ, new_tindex); + } else { + Value *isboxv = ctx.builder.CreateIsNotNull(boxv); + Value *slotv; + MDNode *tbaa; + bool isimmutable; + if (v.ispointer()) { + slotv = v.V; + tbaa = v.tbaa; + isimmutable = v.isimmutable; + } + else { + slotv = emit_static_alloca(ctx, v.V->getType()); + ctx.builder.CreateStore(v.V, slotv); + tbaa = tbaa_stack; + isimmutable = true; + } + slotv = ctx.builder.CreateSelect(isboxv, + decay_derived(boxv), + decay_derived(emit_bitcast(ctx, slotv, boxv->getType()))); + jl_cgval_t newv = jl_cgval_t(slotv, NULL, false, typ, new_tindex); + newv.Vboxed = boxv; + newv.tbaa = tbaa; + newv.isimmutable = isimmutable; + return newv; + } + } + } + else { + return jl_cgval_t(boxed(ctx, v), NULL, true, typ, NULL); + } + return jl_cgval_t(v, typ, new_tindex); +} + // given a value marked with type `v.typ`, compute the mapping and/or boxing to return a value of type `typ` -static jl_cgval_t convert_julia_type(jl_codectx_t &ctx, const jl_cgval_t &v, jl_value_t *typ, bool needsroot) +static jl_cgval_t convert_julia_type(jl_codectx_t &ctx, const jl_cgval_t &v, jl_value_t *typ) { if (typ == (jl_value_t*)jl_typeofbottom_type) return ghostValue(typ); // normalize TypeofBottom to Type{Union{}} @@ -825,8 +976,8 @@ static jl_cgval_t convert_julia_type(jl_codectx_t &ctx, const jl_cgval_t &v, jl_ if (jl_is_leaf_type(typ)) { if (v.TIndex && !isbits_spec(typ)) { // discovered that this union-split type must actually be isboxed - if (v.V) { - return jl_cgval_t(v.V, v.gcroot, true, typ, NULL); + if (v.Vboxed) { + return jl_cgval_t(v.Vboxed, nullptr, true, typ, NULL); } else { // type mismatch: there weren't any boxed values in the union @@ -845,145 +996,7 @@ static jl_cgval_t convert_julia_type(jl_codectx_t &ctx, const jl_cgval_t &v, jl_ else { bool makeboxed = false; if (v.TIndex) { - // previous value was a split union, compute new index, or box - new_tindex = ConstantInt::get(T_int8, 0x80); - SmallBitVector skip_box(1, true); - Value *tindex = ctx.builder.CreateAnd(v.TIndex, ConstantInt::get(T_int8, 0x7f)); - if (jl_is_uniontype(typ)) { - // compute the TIndex mapping from v.typ -> typ - unsigned counter = 0; - for_each_uniontype_small( - // for each old union-split value - [&](unsigned idx, jl_datatype_t *jt) { - unsigned new_idx = get_box_tindex(jt, typ); - bool t; - if (new_idx) { - // found a matching element, - // match it against either the unboxed index - Value *cmp = ctx.builder.CreateICmpEQ(tindex, ConstantInt::get(T_int8, idx)); - new_tindex = ctx.builder.CreateSelect(cmp, ConstantInt::get(T_int8, new_idx), new_tindex); - t = true; - } - else if (!jl_subtype((jl_value_t*)jt, typ)) { - // new value doesn't need to be boxed - // since it isn't part of the new union - t = true; - } - else { - // will actually need to box this element - // since it appeared as a leaftype in the original type - // but not in the remark type - t = false; - } - skip_box.resize(idx + 1, t); - }, - v.typ, - counter); - } - - // some of the values are still unboxed - if (!isa(new_tindex)) { - Value *wasboxed = NULL; - // check if some of the old values might have been boxed - // and copy that information over into the new tindex - if (v.ispointer() && v.V && !isa(v.V)) { - wasboxed = ctx.builder.CreateAnd(v.TIndex, ConstantInt::get(T_int8, 0x80)); - new_tindex = ctx.builder.CreateOr(wasboxed, new_tindex); - wasboxed = ctx.builder.CreateICmpNE(wasboxed, ConstantInt::get(T_int8, 0)); - - // may need to handle compute_box_tindex for some of the values - BasicBlock *currBB = ctx.builder.GetInsertBlock(); - Value *union_box_dt = NULL; - Value *union_box_tindex = ConstantInt::get(T_int8, 0x80); - unsigned counter = 0; - for_each_uniontype_small( - // for each new union-split value - [&](unsigned idx, jl_datatype_t *jt) { - unsigned old_idx = get_box_tindex(jt, v.typ); - if (old_idx == 0) { - if (!union_box_dt) { - BasicBlock *isaBB = BasicBlock::Create(jl_LLVMContext, "union_isa", ctx.f); - ctx.builder.SetInsertPoint(isaBB); - union_box_dt = emit_typeof(ctx, v.V); - } - // didn't handle this item before, select its new union index - Value *cmp = ctx.builder.CreateICmpEQ(maybe_decay_untracked(literal_pointer_val(ctx, (jl_value_t*)jt)), union_box_dt); - union_box_tindex = ctx.builder.CreateSelect(cmp, ConstantInt::get(T_int8, 0x80 | idx), union_box_tindex); - } - }, - typ, - counter); - if (union_box_dt) { - BasicBlock *isaBB = ctx.builder.GetInsertBlock(); - BasicBlock *postBB = BasicBlock::Create(jl_LLVMContext, "post_union_isa", ctx.f); - ctx.builder.CreateBr(postBB); - ctx.builder.SetInsertPoint(currBB); - Value *wasunknown = ctx.builder.CreateICmpEQ(v.TIndex, ConstantInt::get(T_int8, 0x80)); - ctx.builder.CreateCondBr(wasunknown, isaBB, postBB); - ctx.builder.SetInsertPoint(postBB); - PHINode *tindex_phi = ctx.builder.CreatePHI(T_int8, 2); - tindex_phi->addIncoming(new_tindex, currBB); - tindex_phi->addIncoming(union_box_tindex, isaBB); - new_tindex = tindex_phi; - } - - } - - if (!skip_box.all()) { - // some values weren't unboxed in the new union - // box them now (tindex above already selected 0x80 = box for them) - // root the result, and return a new mark_julia_slot over the result - Value *boxv = box_union(ctx, v, skip_box); - Value *froot = NULL; - if (needsroot) { - // build a new gc-root, as needed - froot = emit_local_root(ctx); - Value *newroot = boxv; - if (wasboxed || v.gcroot) { // oldbox might be all ghost values (which don't need roots) - // store either the old box or the new box into the gc-root (skip_box ensures these are mutually-exclusive) - // need to clone the value from `v.gcroot` if this isn't a new box - Value *oldroot; - if (v.gcroot) - oldroot = ctx.builder.CreateLoad(v.gcroot); - else - oldroot = v.V; - newroot = ctx.builder.CreateSelect(wasboxed, emit_bitcast(ctx, oldroot, boxv->getType()), newroot); - } - ctx.builder.CreateStore(newroot, froot); - } - if (v.V == NULL) { - // v.V might be NULL if it was all ghost objects before - return jl_cgval_t(boxv, froot, false, typ, new_tindex); - } - else { - Value *isboxv = ctx.builder.CreateIsNotNull(boxv); - Value *slotv; - MDNode *tbaa; - bool isimmutable; - if (v.ispointer()) { - slotv = v.V; - tbaa = v.tbaa; - isimmutable = v.isimmutable; - } - else { - slotv = emit_static_alloca(ctx, v.V->getType()); - ctx.builder.CreateStore(v.V, slotv); - tbaa = tbaa_stack; - isimmutable = true; - } - slotv = ctx.builder.CreateSelect(isboxv, - decay_derived(boxv), emit_bitcast(ctx, slotv, boxv->getType())); - jl_cgval_t newv = jl_cgval_t(slotv, froot, false, typ, new_tindex); - newv.tbaa = tbaa; - newv.isimmutable = isimmutable; - return newv; - } - } - } - else { - new_tindex = NULL; - makeboxed = true; - } + return convert_julia_type_union(ctx, v, typ); } else if (!v.isboxed && jl_is_uniontype(typ)) { // previous value was unboxed (leaftype), statically compute union tindex @@ -1981,20 +1994,6 @@ static void simple_use_analysis(jl_codectx_t &ctx, jl_value_t *expr) // ---- Get Element Pointer (GEP) instructions within the GC frame ---- -// Emit a gc-root slot indicator -static Value *emit_local_root(jl_codectx_t &ctx, jl_varinfo_t *vi) -{ - Instruction *newroot = new AllocaInst(T_prjlvalue, 0, "gcroot", /*InsertBefore*/ctx.ptlsStates); - if (vi) { - vi->boxroot->replaceAllUsesWith(newroot); - newroot->takeName(vi->boxroot); - vi->boxroot->eraseFromParent(); - vi->boxroot = newroot; - } - - return newroot; -} - static void jl_add_method_root(jl_codectx_t &ctx, jl_value_t *val) { if (jl_is_leaf_type(val) || jl_is_bool(val) || jl_is_symbol(val) || @@ -2062,11 +2061,48 @@ static jl_cgval_t emit_getfield(jl_codectx_t &ctx, const jl_cgval_t &strct, jl_s return mark_julia_type(ctx, result, true, jl_any_type); } +static Value *emit_bits_compare(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgval_t &arg2); + +static Value *emit_bitsunion_compare(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgval_t &arg2) +{ + assert(arg1.typ == arg2.typ && arg1.TIndex && arg2.TIndex && jl_is_uniontype(arg1.typ) && "unimplemented"); + Value *tindex = arg1.TIndex; + BasicBlock *defaultBB = BasicBlock::Create(jl_LLVMContext, "unionbits_is_boxed", ctx.f); + SwitchInst *switchInst = ctx.builder.CreateSwitch(tindex, defaultBB); + BasicBlock *postBB = BasicBlock::Create(jl_LLVMContext, "post_unionbits_is", ctx.f); + ctx.builder.SetInsertPoint(postBB); + PHINode *phi = ctx.builder.CreatePHI(T_int1, 2); + unsigned counter = 0; + for_each_uniontype_small( + [&](unsigned idx, jl_datatype_t *jt) { + BasicBlock *tempBB = BasicBlock::Create(jl_LLVMContext, "unionbits_is", ctx.f); + ctx.builder.SetInsertPoint(tempBB); + switchInst->addCase(ConstantInt::get(T_int8, idx), tempBB); + jl_cgval_t sel_arg1(arg1, (jl_value_t*)jt, NULL); + jl_cgval_t sel_arg2(arg2, (jl_value_t*)jt, NULL); + phi->addIncoming(emit_bits_compare(ctx, sel_arg1, sel_arg2), tempBB); + ctx.builder.CreateBr(postBB); + }, + arg1.typ, + counter); + ctx.builder.SetInsertPoint(defaultBB); + Function *trap_func = Intrinsic::getDeclaration( + ctx.f->getParent(), + Intrinsic::trap); + ctx.builder.CreateCall(trap_func); + ctx.builder.CreateUnreachable(); + ctx.builder.SetInsertPoint(postBB); + return ctx.builder.CreateAnd(phi, ctx.builder.CreateICmpEQ(arg1.TIndex, arg2.TIndex)); +} + static Value *emit_bits_compare(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgval_t &arg2) { assert(jl_is_datatype(arg1.typ) && arg1.typ == arg2.typ); Type *at = julia_type_to_llvm(arg1.typ); + if (type_is_ghost(at)) + return ConstantInt::get(T_int1, 1); + if (at->isIntegerTy() || at->isPointerTy() || at->isFloatingPointTy()) { Type *at_int = INTT(at); Value *varg1 = emit_unbox(ctx, at_int, arg1, arg1.typ); @@ -2115,11 +2151,29 @@ static Value *emit_bits_compare(jl_codectx_t &ctx, const jl_cgval_t &arg1, const Value *subAns, *fld1, *fld2; fld1 = ctx.builder.CreateConstGEP2_32(at, varg1, 0, i); fld2 = ctx.builder.CreateConstGEP2_32(at, varg2, 0, i); - if (type_is_ghost(fld1->getType()->getPointerElementType())) + Type *at_i = cast(fld1)->getResultElementType(); + if (type_is_ghost(at_i)) continue; - subAns = emit_bits_compare(ctx, - mark_julia_slot(fld1, fldty, NULL, arg1.tbaa), - mark_julia_slot(fld2, fldty, NULL, arg2.tbaa)); + if (jl_is_uniontype(fldty)) { + unsigned tindex_offset = cast(at_i)->getNumElements() - 1; + Value *ptindex1 = ctx.builder.CreateConstInBoundsGEP2_32( + at_i, fld1, 0, tindex_offset); + Value *ptindex2 = ctx.builder.CreateConstInBoundsGEP2_32( + at_i, fld2, 0, tindex_offset); + Value *tindex1 = ctx.builder.CreateNUWAdd(ConstantInt::get(T_int8, 1), + ctx.builder.CreateLoad(T_int8, ptindex1)); + Value *tindex2 = ctx.builder.CreateNUWAdd(ConstantInt::get(T_int8, 1), + ctx.builder.CreateLoad(T_int8, ptindex2)); + subAns = emit_bitsunion_compare(ctx, + mark_julia_slot(fld1, fldty, tindex1, arg1.tbaa), + mark_julia_slot(fld2, fldty, tindex2, arg2.tbaa)); + } + else { + assert(jl_is_leaf_type(fldty)); + subAns = emit_bits_compare(ctx, + mark_julia_slot(fld1, fldty, NULL, arg1.tbaa), + mark_julia_slot(fld2, fldty, NULL, arg2.tbaa)); + } answer = ctx.builder.CreateAnd(answer, subAns); } return answer; @@ -2183,6 +2237,9 @@ static Value *emit_f_is(jl_codectx_t &ctx, const jl_cgval_t &arg1, const jl_cgva return cmp; } + // if (arg1.tindex || arg2.tindex) + // TODO: handle with emit_bitsunion_compare + int ptr_comparable = 0; // whether this type is unique'd by pointer if (rt1 == (jl_value_t*)jl_sym_type || rt2 == (jl_value_t*)jl_sym_type) ptr_comparable = 1; @@ -2399,7 +2456,8 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, Value *selidx = ctx.builder.CreateMul(emit_arraylen_prim(ctx, ary), nbytes); selidx = ctx.builder.CreateAdd(selidx, idx); Value *ptindex = ctx.builder.CreateGEP(T_int8, data, selidx); - Value *tindex = ctx.builder.CreateNUWAdd(ConstantInt::get(T_int8, 1), tbaa_decorate(tbaa_arrayselbyte, ctx.builder.CreateLoad(T_int8, ptindex))); + Value *tindex = ctx.builder.CreateNUWAdd(ConstantInt::get(T_int8, 1), + tbaa_decorate(tbaa_arrayselbyte, ctx.builder.CreateLoad(T_int8, ptindex))); Type *AT = ArrayType::get(IntegerType::get(jl_LLVMContext, 8 * al), (elsz + al - 1) / al); AllocaInst *lv = emit_static_alloca(ctx, AT); if (al > 1) @@ -2905,9 +2963,7 @@ static jl_cgval_t emit_call_function_object(jl_method_instance_t *li, jl_llvm_fu jlretty, tindex, tbaa_stack); - // root this, if the return value was a box (tindex & 0x80) != 0 - retval.gcroot = emit_local_root(ctx); - ctx.builder.CreateStore(box, retval.gcroot); + retval.Vboxed = box; break; } case jl_returninfo_t::Ghosts: @@ -3272,7 +3328,6 @@ static jl_cgval_t emit_local(jl_codectx_t &ctx, jl_value_t *slotload) if (vi.boxroot != NULL) { Value *boxed = ctx.builder.CreateLoad(vi.boxroot, vi.isVolatile); Value *box_isnull; - v.gcroot = vi.boxroot; if (vi.usedUndef) box_isnull = ctx.builder.CreateICmpNE(boxed, maybe_decay_untracked(V_null)); if (vi.pTIndex) { @@ -3284,11 +3339,11 @@ static jl_cgval_t emit_local(jl_codectx_t &ctx, jl_value_t *slotload) if (vi.usedUndef) isnull = ctx.builder.CreateSelect(load_unbox, isnull, box_isnull); if (v.V) { // v.V will be null if it is a union of all ghost values - boxed = decay_derived(boxed); v.V = ctx.builder.CreateSelect(load_unbox, emit_bitcast(ctx, - decay_derived(v.V), boxed->getType()), boxed); + decay_derived(v.V), boxed->getType()), decay_derived(boxed)); } else v.V = boxed; + v.Vboxed = boxed; v = update_julia_type(ctx, v, typ); } else { @@ -3372,26 +3427,15 @@ static void emit_assignment(jl_codectx_t &ctx, jl_value_t *l, jl_value_t *r) size_t min_align; dest = try_emit_union_alloca(ctx, ((jl_uniontype_t*)jt), allunbox, min_align); Value *isboxed = NULL; - if (slot.ispointer() && slot.V != NULL && !isa(slot.V)) { + if (slot.isboxed || slot.Vboxed != nullptr) { isboxed = ctx.builder.CreateICmpNE( ctx.builder.CreateAnd(slot.TIndex, ConstantInt::get(T_int8, 0x80)), ConstantInt::get(T_int8, 0)); } if (dest) emit_unionmove(ctx, dest, slot, isboxed, false, NULL); - Value *gcroot = NULL; if (isboxed) { - Value *box; - if (slot.gcroot) { - gcroot = emit_local_root(ctx); - // This might load the wrong object in general, but if it gets selected, below, - // we know that it was in fact the one we wanted. - box = ctx.builder.CreateLoad(slot.gcroot); - } else { - gcroot = emit_static_alloca(ctx, T_pjlvalue); - box = V_null; - } - ctx.builder.CreateStore(box, gcroot); + Value *box = slot.Vboxed ? slot.Vboxed : V_null; if (dest) // might be all ghost values dest = ctx.builder.CreateSelect(isboxed, decay_derived(box), @@ -3402,8 +3446,9 @@ static void emit_assignment(jl_codectx_t &ctx, jl_value_t *l, jl_value_t *r) else { assert(allunbox && "Failed to allocate correct union-type storage."); } + Value *box = slot.Vboxed; slot = mark_julia_slot(dest, slot.typ, slot.TIndex, tbaa_stack); - slot.gcroot = gcroot; + slot.Vboxed = box; } else { bool isboxed; @@ -3449,14 +3494,9 @@ static void emit_assignment(jl_codectx_t &ctx, jl_value_t *l, jl_value_t *r) if (!vi.used) return; - bool needs_root = false; - if ((!vi.isSA && rval_info.gcroot) || !rval_info.isboxed) - // rval needed a gcroot, so lval will need one too - needs_root = true; - // convert rval-type to lval-type jl_value_t *slot_type = vi.value.typ; - rval_info = convert_julia_type(ctx, rval_info, slot_type, /*needs-root*/true); + rval_info = convert_julia_type(ctx, rval_info, slot_type); if (rval_info.typ == jl_bottom_type) return; @@ -3491,18 +3531,13 @@ static void emit_assignment(jl_codectx_t &ctx, jl_value_t *l, jl_value_t *r) // store boxed variables Value *isboxed = NULL; if (vi.boxroot) { - if (isa(vi.boxroot) && needs_root) - emit_local_root(ctx, &vi); // promote variable slot to a gcroot Value *rval; if (vi.pTIndex && rval_info.TIndex) { ctx.builder.CreateStore(rval_info.TIndex, vi.pTIndex, vi.isVolatile); isboxed = ctx.builder.CreateICmpNE( ctx.builder.CreateAnd(rval_info.TIndex, ConstantInt::get(T_int8, 0x80)), ConstantInt::get(T_int8, 0)); - rval = maybe_decay_untracked(V_null); - if (rval_info.ispointer() && rval_info.V != NULL && !isa(rval_info.V) && - !(isa(isboxed) && cast(isboxed)->isZero())) // might be all ghost values or otherwise definitely not boxed - rval = ctx.builder.CreateLoad(rval_info.gcroot); + rval = maybe_decay_untracked(rval_info.Vboxed ? rval_info.Vboxed : V_null); assert(!vi.value.constant); } else { @@ -4576,8 +4611,7 @@ static Function *gen_jlcall_wrapper(jl_method_instance_t *lam, const jl_returnin jlretty, ctx.builder.CreateExtractValue(call, 1), tbaa_stack); - retval.gcroot = emit_local_root(ctx); - ctx.builder.CreateStore(ctx.builder.CreateExtractValue(call, 0), retval.gcroot); + retval.Vboxed = ctx.builder.CreateExtractValue(call, 0); break; case jl_returninfo_t::Ghosts: retval = mark_julia_slot(NULL, jlretty, call, tbaa_stack); @@ -5254,10 +5288,8 @@ static std::unique_ptr emit_function( } } else { - Value *argp = boxed(ctx, theArg); // skip the temporary gcroot since it would be folded to argp anyways + Value *argp = boxed(ctx, theArg); ctx.builder.CreateStore(argp, vi.boxroot); - if (!theArg.isboxed) - emit_local_root(ctx, &vi); // create a root for vi } // get arrayvar data if applicable if (arrayvars.find(i) != arrayvars.end()) { @@ -5284,7 +5316,6 @@ static std::unique_ptr emit_function( ConstantInt::get(T_int32, nreq - 1)) }); restTuple->setAttributes(jltuple_func->getAttributes()); ctx.builder.CreateStore(restTuple, vi.boxroot); - emit_local_root(ctx, &vi); // create a root for vi } } @@ -5537,7 +5568,7 @@ static std::unique_ptr emit_function( // this is basically a copy of emit_assignment, // but where the assignment slot is the retval jl_cgval_t retvalinfo = emit_expr(ctx, jl_exprarg(expr, 0)); - retvalinfo = convert_julia_type(ctx, retvalinfo, jlrettype, /*needs-root*/false); + retvalinfo = convert_julia_type(ctx, retvalinfo, jlrettype); if (retvalinfo.typ == jl_bottom_type) { ctx.builder.CreateUnreachable(); find_next_stmt(-1); @@ -5572,20 +5603,13 @@ static std::unique_ptr emit_function( } else { data = maybe_decay_untracked(V_null); - if (retvalinfo.ispointer() && !isa(retvalinfo.V)) { + if (retvalinfo.Vboxed) { // also need to account for the possibility the return object is boxed // and avoid / skip copying it to the stack isboxed_union = ctx.builder.CreateICmpNE( ctx.builder.CreateAnd(tindex, ConstantInt::get(T_int8, 0x80)), ConstantInt::get(T_int8, 0)); - // Lift the select, because gcroot may be NULL if - // there's no boxed value. - if (isa(isboxed_union)) - data = cast(isboxed_union)->isZero() ? data : ctx.builder.CreateLoad(retvalinfo.gcroot); - else - data = ctx.builder.CreateSelect(isboxed_union, - ctx.builder.CreateLoad(retvalinfo.gcroot), - data); + data = ctx.builder.CreateSelect(isboxed_union, retvalinfo.Vboxed, data); } } } @@ -6481,7 +6505,7 @@ static void init_julia_llvm_env(Module *m) "julia.gc_use"); add_named_global(gc_use_func, (void*)NULL, /*dllimport*/false); - pointer_from_objref_func = Function::Create(FunctionType::get(T_pjlvalue, + pointer_from_objref_func = Function::Create(FunctionType::get(T_size, ArrayRef(PointerType::get(T_jlvalue, AddressSpace::Derived)), false), Function::ExternalLinkage, "julia.pointer_from_objref"); diff --git a/src/datatype.c b/src/datatype.c index ca7f8a361d624..9fb6b19d5591f 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -490,21 +490,17 @@ typedef struct { int64_t b; } bits128_t; -// Note that this function updates len -static jl_value_t *jl_new_bits_internal(jl_value_t *dt, void *data, size_t *len) +// TODO: do we care that this has invalid alignment assumptions? +JL_DLLEXPORT jl_value_t *jl_new_bits(jl_value_t *dt, void *data) { jl_ptls_t ptls = jl_get_ptls_states(); assert(jl_is_datatype(dt)); jl_datatype_t *bt = (jl_datatype_t*)dt; size_t nb = jl_datatype_size(bt); - if (nb == 0) - return jl_new_struct_uninit(bt); - *len = LLT_ALIGN(*len, jl_datatype_align(bt)); - data = (char*)data + (*len); - *len += nb; + if (nb == 0) return jl_new_struct_uninit(bt); // returns bt->instance if (bt == jl_uint8_type) return jl_box_uint8(*(uint8_t*)data); if (bt == jl_int64_type) return jl_box_int64(*(int64_t*)data); - if (bt == jl_bool_type) return (*(int8_t*)data) ? jl_true:jl_false; + if (bt == jl_bool_type) return (*(int8_t*)data) ? jl_true : jl_false; if (bt == jl_int32_type) return jl_box_int32(*(int32_t*)data); if (bt == jl_float64_type) return jl_box_float64(*(double*)data); @@ -520,12 +516,6 @@ static jl_value_t *jl_new_bits_internal(jl_value_t *dt, void *data, size_t *len) return v; } -JL_DLLEXPORT jl_value_t *jl_new_bits(jl_value_t *bt, void *data) -{ - size_t len = 0; - return jl_new_bits_internal(bt, data, &len); -} - // used by boot.jl JL_DLLEXPORT jl_value_t *jl_typemax_uint(jl_value_t *bt) { @@ -784,6 +774,8 @@ JL_DLLEXPORT jl_value_t *jl_get_nth_field(jl_value_t *v, size_t i) if (jl_is_uniontype(ty)) { uint8_t sel = ((uint8_t*)v)[offs + jl_field_size(st, i) - 1]; ty = jl_nth_union_component(ty, sel); + if (jl_is_datatype_singleton((jl_datatype_t*)ty)) + return ((jl_datatype_t*)ty)->instance; } return jl_new_bits(ty, (char*)v + offs); } diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 98dcb81ba6738..f59169287a0e0 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -146,99 +146,113 @@ static Value *uint_cnvt(jl_codectx_t &ctx, Type *to, Value *x) return ctx.builder.CreateZExt(x, to); } -#if JL_LLVM_VERSION >= 40000 -#define LLVM_FP(a,b) APFloat(a(),b) -#else -#define LLVM_FP(a,b) APFloat(a,b) -#endif -static Constant *julia_const_to_llvm(void *ptr, jl_value_t *bt) +static Constant *julia_const_to_llvm(const void *ptr, jl_datatype_t *bt) { // assumes `jl_isbits(bt)`. // `ptr` can point to a inline field, do not read the tag from it. // make sure to return exactly the type specified by // julia_type_to_llvm as this will be assumed by the callee. - if (bt == (jl_value_t*)jl_bool_type) - return ConstantInt::get(T_int8, (*(uint8_t*)ptr) ? 1 : 0); + if (bt == jl_bool_type) + return ConstantInt::get(T_int8, (*(const uint8_t*)ptr) ? 1 : 0); - if (bt == (jl_value_t*)jl_ssavalue_type) - return NULL; + if (jl_is_vecelement_type((jl_value_t*)bt)) + bt = (jl_datatype_t*)jl_tparam0(bt); + + Type *lt = julia_struct_to_llvm((jl_value_t*)bt, NULL, NULL); - if (jl_is_vecelement_type(bt)) - bt = jl_tparam0(bt); + if (type_is_ghost(lt)) + return UndefValue::get(NoopType); - if (jl_is_cpointer_type(bt)) - return ConstantExpr::getIntToPtr(ConstantInt::get(T_size, *(uintptr_t*)ptr), julia_type_to_llvm(bt)); if (jl_is_primitivetype(bt)) { - int nb = jl_datatype_size(bt); - // TODO: non-power-of-2 size datatypes may not be interpreted correctly on big-endian systems - switch (nb) { - case 1: { - uint8_t data8 = *(uint8_t*)ptr; - return ConstantInt::get(T_int8, data8); - } - case 2: { - uint16_t data16 = *(uint16_t*)ptr; - return ConstantInt::get(T_int16, data16); - } - case 4: { - uint32_t data32 = *(uint32_t*)ptr; - if (bt == (jl_value_t*)jl_float32_type) - return ConstantFP::get(jl_LLVMContext, - LLVM_FP(APFloat::IEEEsingle, - APInt(32, data32))); - return ConstantInt::get(T_int32, data32); + if (lt->isFloatTy()) { + uint32_t data32 = *(const uint32_t*)ptr; + return ConstantFP::get(jl_LLVMContext, + APFloat(lt->getFltSemantics(), APInt(32, data32))); } - case 8: { - uint64_t data64 = *(uint64_t*)ptr; - if (bt == (jl_value_t*)jl_float64_type) - return ConstantFP::get(jl_LLVMContext, - LLVM_FP(APFloat::IEEEdouble, - APInt(64, data64))); - return ConstantInt::get(T_int64, data64); + if (lt->isDoubleTy()) { + uint64_t data64 = *(const uint64_t*)ptr; + return ConstantFP::get(jl_LLVMContext, + APFloat(lt->getFltSemantics(), APInt(64, data64))); } - default: - size_t nw = (nb+sizeof(uint64_t)-1)/sizeof(uint64_t); - uint64_t *data = (uint64_t*)ptr; - APInt val; -#if !defined(_P64) - // malloc may not be 16-byte aligned on P32, - // but we must ensure that llvm's uint64_t reads don't fall - // off the end of a page - // where 16-byte alignment requirement == (8-byte typetag) % (uint64_t ArrayRef access) - if (nb % 16 != 0) { - uint64_t *data_a64 = (uint64_t*)alloca(sizeof(uint64_t)*nw); - memcpy(data_a64, data, nb); - val = APInt(8*nb, ArrayRef(data_a64, nw)); - } - else -#endif - val = APInt(8*nb, ArrayRef(data, nw)); - return ConstantInt::get(IntegerType::get(jl_LLVMContext,8*nb),val); + int nb = jl_datatype_size(bt); + APInt val(8 * nb, 0); + void *bits = const_cast(val.getRawData()); + assert(sys::IsLittleEndianHost); + memcpy(bits, ptr, nb); + if (lt->isFloatingPointTy()) { + return ConstantFP::get(jl_LLVMContext, + APFloat(lt->getFltSemantics(), val)); } + assert(cast(lt)->getBitWidth() == 8u * nb); + return ConstantInt::get(lt, val); } + + CompositeType *lct = cast(lt); size_t nf = jl_datatype_nfields(bt); - Constant **fields = (Constant**)alloca(nf * sizeof(Constant*)); + std::vector fields(nf); for (size_t i = 0; i < nf; i++) { - size_t offs = jl_field_offset((jl_datatype_t*)bt, i); + size_t offs = jl_field_offset(bt, i); + assert(!jl_field_isptr(bt, i)); jl_value_t *ft = jl_field_type(bt, i); - Constant *val = julia_const_to_llvm((char*)ptr + offs, ft); - if (val == NULL) - return NULL; + Type *lft = lct->getTypeAtIndex(i); + const uint8_t *ov = (const uint8_t*)ptr + offs; + Constant *val; + if (jl_is_uniontype(ft)) { + // compute the same type layout as julia_struct_to_llvm + size_t fsz = jl_field_size(bt, i); + size_t al = jl_field_align(bt, i); + uint8_t sel = ((const uint8_t*)ptr)[offs + fsz - 1]; + jl_value_t *active_ty = jl_nth_union_component(ft, sel); + size_t active_sz = jl_datatype_size(active_ty); + ArrayType *aty = cast(cast(lft)->getTypeAtIndex(0u)); + assert(aty->getElementType() == IntegerType::get(jl_LLVMContext, 8 * al) && + aty->getNumElements() == (fsz - 1) / al); + std::vector ArrayElements(0); + for (unsigned j = 0; j < aty->getNumElements(); j++) { + APInt Elem(8 * al, 0); + void *bits = const_cast(Elem.getRawData()); + if (active_sz > al) { + memcpy(bits, ov, al); + active_sz -= al; + } + else if (active_sz > 0) { + memcpy(bits, ov, active_sz); + active_sz = 0; + } + ov += al; + ArrayElements.push_back(ConstantInt::get(aty->getElementType(), Elem)); + } + std::vector Elements(0); + Elements.push_back(ConstantArray::get(aty, ArrayElements)); + unsigned remainder = (fsz - 1) % al; + while (remainder--) { + uint8_t byte; + if (active_sz > 0) { + byte = *ov; + active_sz -= 1; + } + else { + byte = 0; + } + ov += 1; + APInt Elem(8, byte); + Elements.push_back(ConstantInt::get(T_int8, Elem)); + } + Elements.push_back(ConstantInt::get(T_int8, sel)); + val = ConstantStruct::get(cast(lft), Elements); + } + else { + val = julia_const_to_llvm(ov, (jl_datatype_t*)ft); + } fields[i] = val; } - Type *t = julia_struct_to_llvm(bt, NULL, NULL); - if (type_is_ghost(t)) - return UndefValue::get(NoopType); - if (t->isVectorTy()) - return ConstantVector::get(ArrayRef(fields, nf)); - if (StructType *st = dyn_cast(t)) { - return ConstantStruct::get(st, ArrayRef(fields, nf)); - } - else { - ArrayType *at = cast(t); - return ConstantArray::get(at, ArrayRef(fields, nf)); - } + if (lct->isVectorTy()) + return ConstantVector::get(fields); + if (StructType *st = dyn_cast(lct)) + return ConstantStruct::get(st, fields); + ArrayType *at = cast(lct); + return ConstantArray::get(at, fields); } static Constant *julia_const_to_llvm(jl_value_t *e) @@ -250,7 +264,7 @@ static Constant *julia_const_to_llvm(jl_value_t *e) jl_value_t *bt = jl_typeof(e); if (!jl_isbits(bt)) return NULL; - return julia_const_to_llvm(e, bt); + return julia_const_to_llvm(e, (jl_datatype_t*)bt); } static jl_cgval_t ghostValue(jl_value_t *ty); @@ -577,15 +591,15 @@ static jl_cgval_t emit_pointerref(jl_codectx_t &ctx, jl_cgval_t *argv) Value *idx = emit_unbox(ctx, T_size, i, (jl_value_t*)jl_long_type); Value *im1 = ctx.builder.CreateSub(idx, ConstantInt::get(T_size, 1)); - if (!jl_isbits(ety)) { - if (ety == (jl_value_t*)jl_any_type) { - Value *thePtr = emit_unbox(ctx, T_pprjlvalue, e, e.typ); - return mark_julia_type( - ctx, - ctx.builder.CreateAlignedLoad(ctx.builder.CreateGEP(thePtr, im1), align_nb), - true, - ety); - } + if (ety == (jl_value_t*)jl_any_type) { + Value *thePtr = emit_unbox(ctx, T_pprjlvalue, e, e.typ); + return mark_julia_type( + ctx, + ctx.builder.CreateAlignedLoad(ctx.builder.CreateGEP(T_prjlvalue, thePtr, im1), align_nb), + true, + ety); + } + else if (!jl_isbits(ety)) { if (!jl_is_structtype(ety) || jl_is_array_type(ety) || !jl_is_leaf_type(ety)) { emit_error(ctx, "pointerref: invalid pointer type"); return jl_cgval_t(); @@ -597,16 +611,17 @@ static jl_cgval_t emit_pointerref(jl_codectx_t &ctx, jl_cgval_t *argv) im1 = ctx.builder.CreateMul(im1, ConstantInt::get(T_size, LLT_ALIGN(size, jl_datatype_align(ety)))); Value *thePtr = emit_unbox(ctx, T_pint8, e, e.typ); - thePtr = ctx.builder.CreateGEP(emit_bitcast(ctx, thePtr, T_pint8), im1); + thePtr = ctx.builder.CreateGEP(T_int8, emit_bitcast(ctx, thePtr, T_pint8), im1); ctx.builder.CreateMemCpy(emit_bitcast(ctx, strct, T_pint8), thePtr, size, 1); return mark_julia_type(ctx, strct, true, ety); } - - bool isboxed; - Type *ptrty = julia_type_to_llvm(e.typ, &isboxed); - assert(!isboxed); - Value *thePtr = emit_unbox(ctx, ptrty, e, e.typ); - return typed_load(ctx, thePtr, im1, ety, tbaa_data, true, align_nb); + else { + bool isboxed; + Type *ptrty = julia_type_to_llvm(ety, &isboxed); + assert(!isboxed); + Value *thePtr = emit_unbox(ctx, ptrty->getPointerTo(), e, e.typ); + return typed_load(ctx, thePtr, im1, ety, tbaa_data, true, align_nb); + } } static jl_cgval_t emit_runtime_pointerset(jl_codectx_t &ctx, jl_cgval_t *argv) @@ -644,7 +659,15 @@ static jl_cgval_t emit_pointerset(jl_codectx_t &ctx, jl_cgval_t *argv) Value *im1 = ctx.builder.CreateSub(idx, ConstantInt::get(T_size, 1)); Value *thePtr; - if (!jl_isbits(ety) && ety != (jl_value_t*)jl_any_type) { + if (ety == (jl_value_t*)jl_any_type) { + // unsafe_store to Ptr{Any} is allowed to implicitly drop GC roots. + thePtr = emit_unbox(ctx, T_psize, e, e.typ); + Instruction *store = ctx.builder.CreateAlignedStore( + emit_pointer_from_objref(ctx, boxed(ctx, x)), + ctx.builder.CreateGEP(T_size, thePtr, im1), align_nb); + tbaa_decorate(tbaa_data, store); + } + else if (!jl_isbits(ety)) { if (!jl_is_structtype(ety) || jl_is_array_type(ety) || !jl_is_leaf_type(ety)) { emit_error(ctx, "pointerset: invalid pointer type"); return jl_cgval_t(); @@ -653,23 +676,15 @@ static jl_cgval_t emit_pointerset(jl_codectx_t &ctx, jl_cgval_t *argv) uint64_t size = jl_datatype_size(ety); im1 = ctx.builder.CreateMul(im1, ConstantInt::get(T_size, LLT_ALIGN(size, jl_datatype_align(ety)))); - ctx.builder.CreateMemCpy(ctx.builder.CreateGEP(thePtr, im1), + ctx.builder.CreateMemCpy(ctx.builder.CreateGEP(T_int8, thePtr, im1), data_pointer(ctx, x, T_pint8), size, align_nb); } else { bool isboxed; - Type *ptrty = julia_type_to_llvm(e.typ, &isboxed); + Type *ptrty = julia_type_to_llvm(ety, &isboxed); assert(!isboxed); - thePtr = emit_unbox(ctx, ptrty, e, e.typ); - if (ety == (jl_value_t*)jl_any_type) { - // unsafe_store to Ptr{Any} is allowed to implicitly drop GC roots. - Instruction *store = ctx.builder.CreateAlignedStore( - emit_pointer_from_objref(ctx, boxed(ctx, x)), - ctx.builder.CreateGEP(thePtr, im1), align_nb); - tbaa_decorate(tbaa_data, store); - } else { - typed_store(ctx, thePtr, im1, x, ety, tbaa_data, NULL, align_nb); - } + thePtr = emit_unbox(ctx, ptrty->getPointerTo(), e, e.typ); + typed_store(ctx, thePtr, im1, x, ety, tbaa_data, NULL, align_nb); } return mark_julia_type(ctx, thePtr, false, aty); } diff --git a/src/llvm-alloc-opt.cpp b/src/llvm-alloc-opt.cpp index 51d9403c96342..d559de07de7eb 100644 --- a/src/llvm-alloc-opt.cpp +++ b/src/llvm-alloc-opt.cpp @@ -491,6 +491,7 @@ void AllocOpt::replaceUsesWith(Instruction *orig_inst, Instruction *new_inst, } else if (auto call = dyn_cast(user)) { if (ptr_from_objref && ptr_from_objref == call->getCalledFunction()) { + new_i = new PtrToIntInst(new_i, T_size, "", call); call->replaceAllUsesWith(new_i); call->eraseFromParent(); return; diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp index 4e15b2955f4f5..6c62c73809bc6 100644 --- a/src/llvm-late-gc-lowering.cpp +++ b/src/llvm-late-gc-lowering.cpp @@ -1156,10 +1156,9 @@ bool LateLowerGCFrame::CleanupIR(Function &F) { (gc_use_func != nullptr && callee == gc_use_func)) { /* No replacement */ } else if (pointer_from_objref_func != nullptr && callee == pointer_from_objref_func) { - auto *ASCI = new AddrSpaceCastInst(CI->getOperand(0), - CI->getType(), "", CI); - ASCI->takeName(CI); - CI->replaceAllUsesWith(ASCI); + auto *ptr = new PtrToIntInst(CI->getOperand(0), CI->getType(), "", CI); + ptr->takeName(CI); + CI->replaceAllUsesWith(ptr); } else if (alloc_obj_func && callee == alloc_obj_func) { assert(CI->getNumArgOperands() == 3); auto sz = (size_t)cast(CI->getArgOperand(1))->getZExtValue(); diff --git a/src/rtutils.c b/src/rtutils.c index e5b4cc4f645a8..1d0e5417163b5 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -887,9 +887,12 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt n += jl_static_show_x(out, *(jl_value_t**)fld_ptr, depth); } else { - n += jl_static_show_x_(out, (jl_value_t*)fld_ptr, - (jl_datatype_t*)jl_field_type(vt, i), - depth); + jl_datatype_t *ft = (jl_datatype_t*)jl_field_type(vt, i); + if (jl_is_uniontype(ft)) { + uint8_t sel = ((uint8_t*)fld_ptr)[jl_field_size(vt, i) - 1]; + ft = (jl_datatype_t*)jl_nth_union_component((jl_value_t*)ft, sel); + } + n += jl_static_show_x_(out, (jl_value_t*)fld_ptr, ft, depth); } if (istuple && tlen == 1) n += jl_printf(out, ","); diff --git a/test/core.jl b/test/core.jl index dfdbfb13513f0..dca835dfc0a49 100644 --- a/test/core.jl +++ b/test/core.jl @@ -5316,6 +5316,49 @@ x.u = initvalue2(Base.uniontypes(U)[1]) x.u = initvalue(Base.uniontypes(U)[2]) @test x.u === initvalue(Base.uniontypes(U)[2]) +# PR #23367 +struct A23367 + x::Union{Int8, Int16, NTuple{7, Int8}, Void} +end +struct B23367 + x::Int8 + y::A23367 + z::Int8 +end +@noinline compare(a, b) = (a === b) # test code-generation of `is` +@noinline get_x(a::A23367) = a.x +function constant23367 end +let + b = B23367(91, A23367(ntuple(i -> Int8(i), Val(7))), 23) + @eval @noinline constant23367(a, b) = (a ? b : $b) + b2 = Ref(b)[] # copy b via field assignment + b3 = B23367[b][1] # copy b via array assignment + @test pointer_from_objref(b) == pointer_from_objref(b) + @test pointer_from_objref(b) != pointer_from_objref(b2) + @test pointer_from_objref(b) != pointer_from_objref(b3) + @test pointer_from_objref(b2) != pointer_from_objref(b3) + + @test b === b2 === b3 + @test compare(b, b2) + @test compare(b, b3) + @test object_id(b) === object_id(b2) == object_id(b3) + @test b.x === Int8(91) + @test b.z === Int8(23) + @test b.y === A23367((Int8(1), Int8(2), Int8(3), Int8(4), Int8(5), Int8(6), Int8(7))) + @test sizeof(b) == 12 + @test A23367(Int8(1)).x === Int8(1) + @test A23367(Int8(0)).x === Int8(0) + @test A23367(Int16(1)).x === Int16(1) + @test A23367(nothing).x === nothing + @test sizeof(b.y) == 8 + @test get_x(A23367(Int8(1))) === Int8(1) + + # test code-generation of constants + other = B23367(91, A23367(nothing), 23) + @test constant23367(true, other) === other + @test constant23367(false, other) === b +end + for U in boxedunions local U for N in (1, 2, 3, 4) @@ -5331,10 +5374,12 @@ for U in boxedunions end # unsafe_wrap -A4 = [1, 2, 3] -@test_throws ArgumentError unsafe_wrap(Array, convert(Ptr{Union{Int, Void}}, pointer(A4)), 3) -A5 = [1 2 3; 4 5 6] -@test_throws ArgumentError unsafe_wrap(Array, convert(Ptr{Union{Int, Void}}, pointer(A5)), 6) +let + A4 = [1, 2, 3] + @test_throws ArgumentError unsafe_wrap(Array, convert(Ptr{Union{Int, Void}}, pointer(A4)), 3) + A5 = [1 2 3; 4 5 6] + @test_throws ArgumentError unsafe_wrap(Array, convert(Ptr{Union{Int, Void}}, pointer(A5)), 6) +end # copy! A23567 = Vector{Union{Float64, Void}}(5) diff --git a/test/libgit2.jl b/test/libgit2.jl index 72c964cb9be0a..d93bb3615c177 100644 --- a/test/libgit2.jl +++ b/test/libgit2.jl @@ -76,6 +76,8 @@ end a = Base.cconvert(Ptr{LibGit2.StrArrayStruct}, p) b = Base.unsafe_convert(Ptr{LibGit2.StrArrayStruct}, a) @test p == convert(Vector{String}, unsafe_load(b)) + @noinline gcuse(a) = a + gcuse(a) end @testset "Signature" begin