From f6518663878bbef442af566a3e91e3502d087f2c Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 11 Jun 2022 16:59:49 -0400 Subject: [PATCH 1/3] Prefix Float16 intrinsics --- src/APInt-C.cpp | 6 ++-- src/julia.expmap | 6 ---- src/julia_internal.h | 14 +++++++-- src/runtime_intrinsics.c | 64 +++++++++++++++++++++++----------------- 4 files changed, 52 insertions(+), 38 deletions(-) diff --git a/src/APInt-C.cpp b/src/APInt-C.cpp index bc0a62e21dd3e..f06d4362bf958 100644 --- a/src/APInt-C.cpp +++ b/src/APInt-C.cpp @@ -316,7 +316,7 @@ void LLVMByteSwap(unsigned numbits, integerPart *pa, integerPart *pr) { void LLVMFPtoInt(unsigned numbits, void *pa, unsigned onumbits, integerPart *pr, bool isSigned, bool *isExact) { double Val; if (numbits == 16) - Val = __gnu_h2f_ieee(*(uint16_t*)pa); + Val = julia__gnu_h2f_ieee(*(uint16_t*)pa); else if (numbits == 32) Val = *(float*)pa; else if (numbits == 64) @@ -391,7 +391,7 @@ void LLVMSItoFP(unsigned numbits, integerPart *pa, unsigned onumbits, integerPar val = a.roundToDouble(true); } if (onumbits == 16) - *(uint16_t*)pr = __gnu_f2h_ieee(val); + *(uint16_t*)pr = julia__gnu_f2h_ieee(val); else if (onumbits == 32) *(float*)pr = val; else if (onumbits == 64) @@ -408,7 +408,7 @@ void LLVMUItoFP(unsigned numbits, integerPart *pa, unsigned onumbits, integerPar val = a.roundToDouble(false); } if (onumbits == 16) - *(uint16_t*)pr = __gnu_f2h_ieee(val); + *(uint16_t*)pr = julia__gnu_f2h_ieee(val); else if (onumbits == 32) *(float*)pr = val; else if (onumbits == 64) diff --git a/src/julia.expmap b/src/julia.expmap index 4b4a792de52b9..41299aa808572 100644 --- a/src/julia.expmap +++ b/src/julia.expmap @@ -38,12 +38,6 @@ environ; __progname; - /* compiler run-time intrinsics */ - __gnu_h2f_ieee; - __extendhfsf2; - __gnu_f2h_ieee; - __truncdfhf2; - local: *; }; diff --git a/src/julia_internal.h b/src/julia_internal.h index 60583f2240aea..384ad9ccb0189 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -1543,8 +1543,18 @@ jl_sym_t *_jl_symbol(const char *str, size_t len) JL_NOTSAFEPOINT; #define JL_GC_ASSERT_LIVE(x) (void)(x) #endif -float __gnu_h2f_ieee(uint16_t param) JL_NOTSAFEPOINT; -uint16_t __gnu_f2h_ieee(float param) JL_NOTSAFEPOINT; +JL_DLLEXPORT float julia__gnu_h2f_ieee(uint16_t param) JL_NOTSAFEPOINT; +JL_DLLEXPORT uint16_t julia__gnu_f2h_ieee(float param) JL_NOTSAFEPOINT; +JL_DLLEXPORT uint16_t julia__truncdfhf2(double param) JL_NOTSAFEPOINT; +//JL_DLLEXPORT double julia__extendhfdf2(uint16_t n) JL_NOTSAFEPOINT; +//JL_DLLEXPORT int32_t julia__fixhfsi(uint16_t n) JL_NOTSAFEPOINT; +//JL_DLLEXPORT int64_t julia__fixhfdi(uint16_t n) JL_NOTSAFEPOINT; +//JL_DLLEXPORT uint32_t julia__fixunshfsi(uint16_t n) JL_NOTSAFEPOINT; +//JL_DLLEXPORT uint64_t julia__fixunshfdi(uint16_t n) JL_NOTSAFEPOINT; +//JL_DLLEXPORT uint16_t julia__floatsihf(int32_t n) JL_NOTSAFEPOINT; +//JL_DLLEXPORT uint16_t julia__floatdihf(int64_t n) JL_NOTSAFEPOINT; +//JL_DLLEXPORT uint16_t julia__floatunsihf(uint32_t n) JL_NOTSAFEPOINT; +//JL_DLLEXPORT uint16_t julia__floatundihf(uint64_t n) JL_NOTSAFEPOINT; #ifdef __cplusplus } diff --git a/src/runtime_intrinsics.c b/src/runtime_intrinsics.c index 89c9449e55920..ea912b61ac4c3 100644 --- a/src/runtime_intrinsics.c +++ b/src/runtime_intrinsics.c @@ -15,9 +15,6 @@ const unsigned int host_char_bit = 8; // float16 intrinsics -// TODO: use LLVM's compiler-rt on all platforms (Xcode already links compiler-rt) - -#if !defined(_OS_DARWIN_) static inline float half_to_float(uint16_t ival) JL_NOTSAFEPOINT { @@ -188,22 +185,17 @@ static inline uint16_t float_to_half(float param) JL_NOTSAFEPOINT return h; } -JL_DLLEXPORT float __gnu_h2f_ieee(uint16_t param) +JL_DLLEXPORT float julia__gnu_h2f_ieee(uint16_t param) { return half_to_float(param); } -JL_DLLEXPORT float __extendhfsf2(uint16_t param) -{ - return half_to_float(param); -} - -JL_DLLEXPORT uint16_t __gnu_f2h_ieee(float param) +JL_DLLEXPORT uint16_t julia__gnu_f2h_ieee(float param) { return float_to_half(param); } -JL_DLLEXPORT uint16_t __truncdfhf2(double param) +JL_DLLEXPORT uint16_t julia__truncdfhf2(double param) { float res = (float)param; uint32_t resi; @@ -225,7 +217,25 @@ JL_DLLEXPORT uint16_t __truncdfhf2(double param) return float_to_half(res); } -#endif +//JL_DLLEXPORT double julia__extendhfdf2(uint16_t n) { return (double)julia__gnu_h2f_ieee(n); } +//JL_DLLEXPORT int32_t julia__fixhfsi(uint16_t n) { return (int32_t)julia__gnu_h2f_ieee(n); } +//JL_DLLEXPORT int64_t julia__fixhfdi(uint16_t n) { return (int64_t)julia__gnu_h2f_ieee(n); } +//JL_DLLEXPORT uint32_t julia__fixunshfsi(uint16_t n) { return (uint32_t)julia__gnu_h2f_ieee(n); } +//JL_DLLEXPORT uint64_t julia__fixunshfdi(uint16_t n) { return (uint64_t)julia__gnu_h2f_ieee(n); } +//JL_DLLEXPORT uint16_t julia__floatsihf(int32_t n) { return julia__gnu_f2h_ieee((float)n); } +//JL_DLLEXPORT uint16_t julia__floatdihf(int64_t n) { return julia__gnu_f2h_ieee((float)n); } +//JL_DLLEXPORT uint16_t julia__floatunsihf(uint32_t n) { return julia__gnu_f2h_ieee((float)n); } +//JL_DLLEXPORT uint16_t julia__floatundihf(uint64_t n) { return julia__gnu_f2h_ieee((float)n); } +//HANDLE_LIBCALL(F16, F128, __extendhftf2) +//HANDLE_LIBCALL(F16, F80, __extendhfxf2) +//HANDLE_LIBCALL(F80, F16, __truncxfhf2) +//HANDLE_LIBCALL(F128, F16, __trunctfhf2) +//HANDLE_LIBCALL(PPCF128, F16, __trunctfhf2) +//HANDLE_LIBCALL(F16, I128, __fixhfti) +//HANDLE_LIBCALL(F16, I128, __fixunshfti) +//HANDLE_LIBCALL(I128, F16, __floattihf) +//HANDLE_LIBCALL(I128, F16, __floatuntihf) + // run time version of bitcast intrinsic JL_DLLEXPORT jl_value_t *jl_bitcast(jl_value_t *ty, jl_value_t *v) @@ -551,9 +561,9 @@ static inline unsigned select_by_size(unsigned sz) JL_NOTSAFEPOINT } #define fp_select(a, func) \ - sizeof(a) == sizeof(float) ? func##f((float)a) : func(a) + sizeof(a) <= sizeof(float) ? func##f((float)a) : func(a) #define fp_select2(a, b, func) \ - sizeof(a) == sizeof(float) ? func##f(a, b) : func(a, b) + sizeof(a) <= sizeof(float) ? func##f(a, b) : func(a, b) // fast-function generators // @@ -597,11 +607,11 @@ static inline void name(unsigned osize, void *pa, void *pr) JL_NOTSAFEPOINT \ static inline void name(unsigned osize, void *pa, void *pr) JL_NOTSAFEPOINT \ { \ uint16_t a = *(uint16_t*)pa; \ - float A = __gnu_h2f_ieee(a); \ + float A = julia__gnu_h2f_ieee(a); \ if (osize == 16) { \ float R; \ OP(&R, A); \ - *(uint16_t*)pr = __gnu_f2h_ieee(R); \ + *(uint16_t*)pr = julia__gnu_f2h_ieee(R); \ } else { \ OP((uint16_t*)pr, A); \ } \ @@ -625,11 +635,11 @@ static void jl_##name##16(unsigned runtime_nbits, void *pa, void *pb, void *pr) { \ uint16_t a = *(uint16_t*)pa; \ uint16_t b = *(uint16_t*)pb; \ - float A = __gnu_h2f_ieee(a); \ - float B = __gnu_h2f_ieee(b); \ + float A = julia__gnu_h2f_ieee(a); \ + float B = julia__gnu_h2f_ieee(b); \ runtime_nbits = 16; \ float R = OP(A, B); \ - *(uint16_t*)pr = __gnu_f2h_ieee(R); \ + *(uint16_t*)pr = julia__gnu_f2h_ieee(R); \ } // float or integer inputs, bool output @@ -650,8 +660,8 @@ static int jl_##name##16(unsigned runtime_nbits, void *pa, void *pb) JL_NOTSAFEP { \ uint16_t a = *(uint16_t*)pa; \ uint16_t b = *(uint16_t*)pb; \ - float A = __gnu_h2f_ieee(a); \ - float B = __gnu_h2f_ieee(b); \ + float A = julia__gnu_h2f_ieee(a); \ + float B = julia__gnu_h2f_ieee(b); \ runtime_nbits = 16; \ return OP(A, B); \ } @@ -691,12 +701,12 @@ static void jl_##name##16(unsigned runtime_nbits, void *pa, void *pb, void *pc, uint16_t a = *(uint16_t*)pa; \ uint16_t b = *(uint16_t*)pb; \ uint16_t c = *(uint16_t*)pc; \ - float A = __gnu_h2f_ieee(a); \ - float B = __gnu_h2f_ieee(b); \ - float C = __gnu_h2f_ieee(c); \ + float A = julia__gnu_h2f_ieee(a); \ + float B = julia__gnu_h2f_ieee(b); \ + float C = julia__gnu_h2f_ieee(c); \ runtime_nbits = 16; \ float R = OP(A, B, C); \ - *(uint16_t*)pr = __gnu_f2h_ieee(R); \ + *(uint16_t*)pr = julia__gnu_f2h_ieee(R); \ } @@ -1318,7 +1328,7 @@ static inline int fpiseq##nbits(c_type a, c_type b) JL_NOTSAFEPOINT { \ fpiseq_n(float, 32) fpiseq_n(double, 64) #define fpiseq(a,b) \ - sizeof(a) == sizeof(float) ? fpiseq32(a, b) : fpiseq64(a, b) + sizeof(a) <= sizeof(float) ? fpiseq32(a, b) : fpiseq64(a, b) bool_fintrinsic(eq,eq_float) bool_fintrinsic(ne,ne_float) @@ -1367,7 +1377,7 @@ cvt_iintrinsic(LLVMFPtoUI, fptoui) if (!(osize < 8 * sizeof(a))) \ jl_error("fptrunc: output bitsize must be < input bitsize"); \ else if (osize == 16) \ - *(uint16_t*)pr = __gnu_f2h_ieee(a); \ + *(uint16_t*)pr = julia__gnu_f2h_ieee(a); \ else if (osize == 32) \ *(float*)pr = a; \ else if (osize == 64) \ From ff360153dd70491c6ec7812a486c860c54a3f55c Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Sat, 11 Jun 2022 17:26:34 -0400 Subject: [PATCH 2/3] Define aliases to FP16 crt in the OJIT --- src/jitlayers.cpp | 18 ++++++++++++++++-- src/jitlayers.h | 1 + test/intrinsics.jl | 24 ++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 50316c258d3a0..009b969201164 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -1107,12 +1107,26 @@ JuliaOJIT::JuliaOJIT() } JD.addToLinkOrder(GlobalJD, orc::JITDylibLookupFlags::MatchExportedSymbolsOnly); + + orc::SymbolAliasMap jl_crt = { + { mangle("__gnu_h2f_ieee"), { mangle("julia__gnu_h2f_ieee"), JITSymbolFlags::Exported } }, + { mangle("__extendhfsf2"), { mangle("julia__gnu_h2f_ieee"), JITSymbolFlags::Exported } }, + { mangle("__gnu_f2h_ieee"), { mangle("julia__gnu_f2h_ieee"), JITSymbolFlags::Exported } }, + { mangle("__truncsfhf2"), { mangle("julia__gnu_f2h_ieee"), JITSymbolFlags::Exported } }, + { mangle("__truncdfhf2"), { mangle("julia__truncdfhf2"), JITSymbolFlags::Exported } } + }; + cantFail(GlobalJD.define(orc::symbolAliases(jl_crt))); } -void JuliaOJIT::addGlobalMapping(StringRef Name, uint64_t Addr) +orc::SymbolStringPtr JuliaOJIT::mangle(StringRef Name) { std::string MangleName = getMangledName(Name); - cantFail(JD.define(orc::absoluteSymbols({{ES.intern(MangleName), JITEvaluatedSymbol::fromPointer((void*)Addr)}}))); + return ES.intern(MangleName); +} + +void JuliaOJIT::addGlobalMapping(StringRef Name, uint64_t Addr) +{ + cantFail(JD.define(orc::absoluteSymbols({{mangle(Name), JITEvaluatedSymbol::fromPointer((void*)Addr)}}))); } void JuliaOJIT::addModule(orc::ThreadSafeModule TSM) diff --git a/src/jitlayers.h b/src/jitlayers.h index ee3d0c14b3751..c4a89f882beaa 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -369,6 +369,7 @@ class JuliaOJIT { void RegisterJITEventListener(JITEventListener *L); #endif + orc::SymbolStringPtr mangle(StringRef Name); void addGlobalMapping(StringRef Name, uint64_t Addr); void addModule(orc::ThreadSafeModule M); diff --git a/test/intrinsics.jl b/test/intrinsics.jl index 48c5bed6abb36..10d3a5fedc867 100644 --- a/test/intrinsics.jl +++ b/test/intrinsics.jl @@ -169,6 +169,30 @@ end @test_intrinsic Core.Intrinsics.fptoui UInt Float16(3.3) UInt(3) end +if Sys.ARCH == :aarch64 + # On AArch64 we are following the `_Float16` ABI. Buthe these functions expect `Int16`. + # TODO: SHould we have `Chalf == Int16` and `Cfloat16 == Float16`? + extendhfsf2(x::Float16) = ccall("extern __extendhfsf2", llvmcall, Float32, (Int16,), reinterpret(Int16, x)) + gnu_h2f_ieee(x::Float16) = ccall("extern __gnu_h2f_ieee", llvmcall, Float32, (Int16,), reinterpret(Int16, x)) + truncsfhf2(x::Float32) = reinterpret(Float16, ccall("extern __truncsfhf2", llvmcall, Int16, (Float32,), x)) + gnu_f2h_ieee(x::Float32) = reinterpret(Float16, ccall("extern __gnu_f2h_ieee", llvmcall, Int16, (Float32,), x)) + truncdfhf2(x::Float64) = reinterpret(Float16, ccall("extern __truncdfhf2", llvmcall, Int16, (Float64,), x)) +else + extendhfsf2(x::Float16) = ccall("extern __extendhfsf2", llvmcall, Float32, (Float16,), x) + gnu_h2f_ieee(x::Float16) = ccall("extern __gnu_h2f_ieee", llvmcall, Float32, (Float16,), x) + truncsfhf2(x::Float32) = ccall("extern __truncsfhf2", llvmcall, Float16, (Float32,), x) + gnu_f2h_ieee(x::Float32) = ccall("extern __gnu_f2h_ieee", llvmcall, Float16, (Float32,), x) + truncdfhf2(x::Float64) = ccall("extern __truncdfhf2", llvmcall, Float16, (Float64,), x) +end + +@testset "Float16 intrinsics (crt)" begin + @test extendhfsf2(Float16(3.3)) == 3.3007812f0 + @test gnu_h2f_ieee(Float16(3.3)) == 3.3007812f0 + @test truncsfhf2(3.3f0) == Float16(3.3) + @test gnu_f2h_ieee(3.3f0) == Float16(3.3) + @test truncdfhf2(3.3) == Float16(3.3) +end + using Base.Experimental: @force_compile @test_throws ConcurrencyViolationError("invalid atomic ordering") (@force_compile; Core.Intrinsics.atomic_fence(:u)) === nothing @test_throws ConcurrencyViolationError("invalid atomic ordering") (@force_compile; Core.Intrinsics.atomic_fence(Symbol("u", "x"))) === nothing From 3407fb3f7f93e4a372393ab81c7a985a99867c50 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Sun, 12 Jun 2022 11:59:05 -0400 Subject: [PATCH 3/3] Emit aliases into the system image - Put the interposer in llvm.compiler.used. - Injecting the aliases after optimization: Our multiversioning pass interacts badly with the llvm.compiler.used gvar. Co-authored-by: Tim Besard Co-authored-by: Valentin Churavy --- src/aotcompile.cpp | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 13872b29322a3..7a19b34bd6824 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -32,6 +33,7 @@ #include #include #include +#include #include #include #if defined(USE_POLLY) @@ -434,6 +436,23 @@ static void reportWriterError(const ErrorInfoBase &E) jl_safe_printf("ERROR: failed to emit output file %s\n", err.c_str()); } +static void injectCRTAlias(Module &M, StringRef name, StringRef alias, FunctionType *FT) +{ + Function *target = M.getFunction(alias); + if (!target) { + target = Function::Create(FT, Function::ExternalLinkage, alias, M); + } + Function *interposer = Function::Create(FT, Function::WeakAnyLinkage, name, M); + appendToCompilerUsed(M, {interposer}); + + llvm::IRBuilder<> builder(BasicBlock::Create(M.getContext(), "top", interposer)); + SmallVector CallArgs; + for (auto &arg : interposer->args()) + CallArgs.push_back(&arg); + auto val = builder.CreateCall(target, CallArgs); + builder.CreateRet(val); +} + // takes the running content that has collected in the shadow module and dump it to disk // this builds the object file portion of the sysimage files for fast startup @@ -550,7 +569,24 @@ void jl_dump_native_impl(void *native_code, auto add_output = [&] (Module &M, StringRef unopt_bc_Name, StringRef bc_Name, StringRef obj_Name, StringRef asm_Name) { preopt.run(M); optimizer.run(M); + + // We would like to emit an alias or an weakref alias to redirect these symbols + // but LLVM doesn't let us emit a GlobalAlias to a declaration... + // So for now we inject a definition of these functions that calls our runtime + // functions. We do so after optimization to avoid cloning these functions. + injectCRTAlias(M, "__gnu_h2f_ieee", "julia__gnu_h2f_ieee", + FunctionType::get(Type::getFloatTy(Context), { Type::getHalfTy(Context) }, false)); + injectCRTAlias(M, "__extendhfsf2", "julia__gnu_h2f_ieee", + FunctionType::get(Type::getFloatTy(Context), { Type::getHalfTy(Context) }, false)); + injectCRTAlias(M, "__gnu_f2h_ieee", "julia__gnu_f2h_ieee", + FunctionType::get(Type::getHalfTy(Context), { Type::getFloatTy(Context) }, false)); + injectCRTAlias(M, "__truncsfhf2", "julia__gnu_f2h_ieee", + FunctionType::get(Type::getHalfTy(Context), { Type::getFloatTy(Context) }, false)); + injectCRTAlias(M, "__truncdfhf2", "julia__truncdfhf2", + FunctionType::get(Type::getHalfTy(Context), { Type::getDoubleTy(Context) }, false)); + postopt.run(M); + if (unopt_bc_fname) emit_result(unopt_bc_Archive, unopt_bc_Buffer, unopt_bc_Name, outputs); if (bc_fname)