From 18d22054132646f4b2cb688aba86df6c21302eef Mon Sep 17 00:00:00 2001 From: N1ark Date: Sun, 6 Sep 2026 17:57:48 +0100 Subject: [PATCH 1/8] cranelift: use fallbacks for abs f16, f128 --- .../src/codegen_f16_f128.rs | 14 ----------- .../src/intrinsics/mod.rs | 25 ++++++++++--------- 2 files changed, 13 insertions(+), 26 deletions(-) diff --git a/compiler/rustc_codegen_cranelift/src/codegen_f16_f128.rs b/compiler/rustc_codegen_cranelift/src/codegen_f16_f128.rs index 09762f1a451b6..874a68fe5b872 100644 --- a/compiler/rustc_codegen_cranelift/src/codegen_f16_f128.rs +++ b/compiler/rustc_codegen_cranelift/src/codegen_f16_f128.rs @@ -160,20 +160,6 @@ pub(crate) fn neg_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { fx.bcx.ins().bitcast(types::F128, MemFlagsData::new(), bits) } -pub(crate) fn abs_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { - let bits = fx.bcx.ins().bitcast(types::I16, MemFlagsData::new(), value); - let bits = fx.bcx.ins().band_imm_u(bits, 0x7fff); - fx.bcx.ins().bitcast(types::F16, MemFlagsData::new(), bits) -} - -pub(crate) fn abs_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { - let bits = fx.bcx.ins().bitcast(types::I128, MemFlagsData::new(), value); - let (low, high) = fx.bcx.ins().isplit(bits); - let high = fx.bcx.ins().band_imm_u(high, 0x7fff_ffff_ffff_ffff_u64 as i64); - let bits = fx.bcx.ins().iconcat(low, high); - fx.bcx.ins().bitcast(types::F128, MemFlagsData::new(), bits) -} - pub(crate) fn codegen_cast( fx: &mut FunctionCx<'_, '_, '_>, from: Value, diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index cf1c1f027e7f5..257d571d6745c 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -455,6 +455,13 @@ fn codegen_float_intrinsic_call<'tcx>( true } +/// Used to distinguish fallbacks of float intrinsics. For some we have a codegen fallback, +/// while for others we fallback to an external libcall. +enum IntrinsicFallback { + Fallback(&'static str), + Codegen(Value), +} + fn codegen_regular_intrinsic_call<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, instance: Instance<'tcx>, @@ -1176,24 +1183,16 @@ fn codegen_regular_intrinsic_call<'tcx>( let ty::Float(float_ty) = layout.ty.kind() else { span_bug!( source_info.span, - "expected float type for fabs intrinsic: {:?}", + "expected float type for {:?} intrinsic: {:?}", + intrinsic, layout.ty ); }; - enum IntrinsicFallback { - Fallback(&'static str), - Codegen(Value), - } use FloatTy::*; use IntrinsicFallback::*; let x = arg.load_scalar(fx); let res = match (intrinsic, float_ty) { (sym::fabs, F32 | F64) => Codegen(fx.bcx.ins().fabs(x)), - // FIXME(bytecodealliance/wasmtime#8312): Use `fabsf16` once Cranelift - // backend lowerings are implemented. - (sym::fabs, F16) => Codegen(codegen_f16_f128::abs_f16(fx, x)), - (sym::fabs, F128) => Codegen(codegen_f16_f128::abs_f128(fx, x)), - (sym::exp, F32) => Fallback("expf"), (sym::exp, F64) => Fallback("exp"), (sym::exp, F128) => Fallback("expf128"), @@ -1222,8 +1221,10 @@ fn codegen_regular_intrinsic_call<'tcx>( (sym::cos, F64) => Fallback("cos"), (sym::cos, F128) => Fallback("cosf128"), - (_, F16) => { - // We implement fallbacks for other f16 intrinsics via f32 + (sym::fabs, F128) | (_, F16) => { + // FIXME(bytecodealliance/wasmtime#8312): Use the native operations once + // Cranelift backend lowerings for `f16` are implemented. + // We use the intrinsic fallback bodies for the rest return Err(Instance::new_raw(instance.def_id(), instance.args)); } From da81486b761185f246f731e73f55f7d81c4269ca Mon Sep 17 00:00:00 2001 From: N1ark Date: Sun, 6 Sep 2026 19:11:42 +0100 Subject: [PATCH 2/8] Make rounding intrinsics generic --- .../src/compiler_builtins.rs | 4 + .../src/intrinsics/mod.rs | 51 ++-- .../rustc_codegen_gcc/src/intrinsic/mod.rs | 70 +++-- compiler/rustc_codegen_llvm/src/intrinsic.rs | 43 +-- compiler/rustc_codegen_llvm/src/lib.rs | 4 +- .../src/interpret/intrinsics.rs | 124 +------- .../src/interpret/intrinsics/simd.rs | 16 +- .../rustc_hir_analysis/src/check/intrinsic.rs | 57 +--- compiler/rustc_span/src/symbol.rs | 25 +- library/core/src/intrinsics/macros.rs | 40 ++- library/core/src/intrinsics/mod.rs | 266 ++++++------------ library/core/src/num/f128.rs | 10 +- library/core/src/num/f16.rs | 10 +- library/core/src/num/f32.rs | 10 +- library/core/src/num/f64.rs | 10 +- .../core_arch/src/aarch64/neon/generated.rs | 10 +- .../crates/core_arch/src/wasm32/mod.rs | 16 +- .../spec/neon/aarch64.spec.yml | 10 +- src/tools/clippy/clippy_utils/src/sym.rs | 3 - tests/ui/intrinsics/reify-intrinsic.rs | 2 +- tests/ui/intrinsics/reify-intrinsic.stderr | 8 +- 21 files changed, 282 insertions(+), 507 deletions(-) diff --git a/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs b/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs index ca9157daae584..ed5b904fee259 100644 --- a/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs +++ b/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs @@ -172,6 +172,10 @@ builtin_functions! { #[cfg(not(all(target_os = "windows", target_env = "gnu")))] fn rintf128(f: f128) -> f128; #[cfg(not(all(target_os = "windows", target_env = "gnu")))] + fn roundf16(f: f16) -> f16; + #[cfg(not(all(target_os = "windows", target_env = "gnu")))] + fn roundf128(f: f128) -> f128; + #[cfg(not(all(target_os = "windows", target_env = "gnu")))] fn sqrtf16(f: f16) -> f16; #[cfg(not(all(target_os = "windows", target_env = "gnu")))] fn sqrtf128(f: f128) -> f128; diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index 257d571d6745c..3fd8b619f820b 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -354,31 +354,6 @@ fn codegen_float_intrinsic_call<'tcx>( sym::copysignf64 => ("copysign", 2, fx.tcx.types.f64, types::F64), sym::copysignf128 => return false, // has a fallback - sym::floorf16 => return false, // has a fallback via f32 - sym::floorf32 => ("floorf", 1, fx.tcx.types.f32, types::F32), - sym::floorf64 => ("floor", 1, fx.tcx.types.f64, types::F64), - sym::floorf128 => ("floorf128", 1, fx.tcx.types.f128, types::F128), - - sym::ceilf16 => return false, // has a fallback via f32 - sym::ceilf32 => ("ceilf", 1, fx.tcx.types.f32, types::F32), - sym::ceilf64 => ("ceil", 1, fx.tcx.types.f64, types::F64), - sym::ceilf128 => ("ceilf128", 1, fx.tcx.types.f128, types::F128), - - sym::truncf16 => return false, // has a fallback via f32 - sym::truncf32 => ("truncf", 1, fx.tcx.types.f32, types::F32), - sym::truncf64 => ("trunc", 1, fx.tcx.types.f64, types::F64), - sym::truncf128 => ("truncf128", 1, fx.tcx.types.f128, types::F128), - - sym::round_ties_even_f16 => return false, // has a fallback via f32 - sym::round_ties_even_f32 => ("rintf", 1, fx.tcx.types.f32, types::F32), - sym::round_ties_even_f64 => ("rint", 1, fx.tcx.types.f64, types::F64), - sym::round_ties_even_f128 => ("rintf128", 1, fx.tcx.types.f128, types::F128), - - sym::roundf16 => return false, // has a fallback via f32 - sym::roundf32 => ("roundf", 1, fx.tcx.types.f32, types::F32), - sym::roundf64 => ("round", 1, fx.tcx.types.f64, types::F64), - sym::roundf128 => ("roundf128", 1, fx.tcx.types.f128, types::F128), - _ => return false, }; @@ -418,10 +393,6 @@ fn codegen_float_intrinsic_call<'tcx>( fx.bcx.ins().fma(args[0], args[1], args[2]) } sym::copysignf32 | sym::copysignf64 => fx.bcx.ins().fcopysign(args[0], args[1]), - sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(args[0]), - sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(args[0]), - sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(args[0]), - sym::round_ties_even_f32 | sym::round_ties_even_f64 => fx.bcx.ins().nearest(args[0]), sym::sqrtf32 | sym::sqrtf64 => fx.bcx.ins().sqrt(args[0]), // These intrinsics aren't supported natively by Cranelift. @@ -1171,6 +1142,11 @@ fn codegen_regular_intrinsic_call<'tcx>( } sym::fabs + | sym::floor + | sym::ceil + | sym::trunc + | sym::round + | sym::round_ties_even | sym::exp | sym::exp2 | sym::log @@ -1193,6 +1169,23 @@ fn codegen_regular_intrinsic_call<'tcx>( let x = arg.load_scalar(fx); let res = match (intrinsic, float_ty) { (sym::fabs, F32 | F64) => Codegen(fx.bcx.ins().fabs(x)), + + (sym::floor, F32 | F64) => Codegen(fx.bcx.ins().floor(x)), + (sym::floor, F128) => Fallback("floorf128"), + + (sym::ceil, F32 | F64) => Codegen(fx.bcx.ins().ceil(x)), + (sym::ceil, F128) => Fallback("ceilf128"), + + (sym::trunc, F32 | F64) => Codegen(fx.bcx.ins().trunc(x)), + (sym::trunc, F128) => Fallback("truncf128"), + + (sym::round_ties_even, F32 | F64) => Codegen(fx.bcx.ins().nearest(x)), + (sym::round_ties_even, F128) => Fallback("rintf128"), + + (sym::round, F32) => Fallback("roundf"), + (sym::round, F64) => Fallback("round"), + (sym::round, F128) => Fallback("roundf128"), + (sym::exp, F32) => Fallback("expf"), (sym::exp, F64) => Fallback("exp"), (sym::exp, F128) => Fallback("expf128"), diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index 5550d22b33aa3..abff9de8575f0 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -77,18 +77,7 @@ fn get_simple_intrinsic<'gcc, 'tcx>( sym::maximumf128 => return float_intrinsic(cx, cx.type_f128(), "fmaximumf128"), sym::copysignf32 => "copysignf", sym::copysignf64 => "copysign", - sym::floorf32 => "floorf", - sym::floorf64 => "floor", - sym::ceilf32 => "ceilf", - sym::ceilf64 => "ceil", sym::powf128 => return float_intrinsic(cx, cx.type_f128(), "powf128"), - sym::truncf32 => "truncf", - sym::truncf64 => "trunc", - // We match the LLVM backend and lower this to `rint`. - sym::round_ties_even_f32 => "rintf", - sym::round_ties_even_f64 => "rint", - sym::roundf32 => "roundf", - sym::roundf64 => "round", sym::abort => "abort", _ => return None, }; @@ -102,18 +91,18 @@ fn get_simple_function_f128<'gcc, 'tcx>( ) -> Function<'gcc> { let f128_type = cx.type_f128(); let func_name = match name { - sym::ceilf128 => "ceilf128", + sym::ceil => "ceilf128", sym::cos => "cosf128", sym::fabs => "fabsf128", sym::exp => "expf128", sym::exp2 => "exp2f128", - sym::floorf128 => "floorf128", + sym::floor => "floorf128", sym::log => "logf128", sym::log2 => "log2f128", sym::log10 => "log10f128", - sym::truncf128 => "truncf128", - sym::roundf128 => "roundf128", - sym::round_ties_even_f128 => "roundevenf128", + sym::trunc => "truncf128", + sym::round => "roundf128", + sym::round_ties_even => "roundevenf128", sym::sin => "sinf128", sym::sqrtf128 => "sqrtf128", _ => span_bug!(span, "used get_simple_function_f128 for non-unary f128 intrinsic"), @@ -135,22 +124,22 @@ fn f16_builtin<'gcc, 'tcx>( ) -> RValue<'gcc> { let f32_type = cx.type_f32(); let builtin_name = match name { - sym::ceilf16 => "__builtin_ceilf", + sym::ceil => "__builtin_ceilf", sym::copysignf16 => "__builtin_copysignf", sym::cos => "cosf", sym::exp => "expf", sym::exp2 => "exp2f", sym::fabs => "fabsf", - sym::floorf16 => "__builtin_floorf", + sym::floor => "__builtin_floorf", sym::log => "logf", sym::log2 => "log2f", sym::log10 => "log10f", sym::powf16 => "__builtin_powf", - sym::roundf16 => "__builtin_roundf", - sym::round_ties_even_f16 => "__builtin_rintf", + sym::round => "__builtin_roundf", + sym::round_ties_even => "__builtin_rintf", sym::sin => "sinf", sym::sqrtf16 => "__builtin_sqrtf", - sym::truncf16 => "__builtin_truncf", + sym::trunc => "__builtin_truncf", _ => unreachable!(), }; @@ -215,22 +204,8 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc &args.iter().map(|arg| arg.immediate()).collect::>(), ) } - sym::ceilf16 - | sym::copysignf16 - | sym::floorf16 - | sym::powf16 - | sym::roundf16 - | sym::round_ties_even_f16 - | sym::sqrtf16 - | sym::truncf16 => f16_builtin(self, name, args), - sym::ceilf128 - | sym::floorf128 - | sym::truncf128 - | sym::roundf128 - | sym::round_ties_even_f128 - | sym::sqrtf128 - if self.cx.supports_f128_type => - { + sym::copysignf16 | sym::powf16 | sym::sqrtf16 => f16_builtin(self, name, args), + sym::sqrtf128 if self.cx.supports_f128_type => { let func = get_simple_function_f128(span, self, name); self.cx.context.new_call( self.location, @@ -413,6 +388,11 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc } } sym::fabs + | sym::floor + | sym::ceil + | sym::trunc + | sym::round + | sym::round_ties_even | sym::exp | sym::exp2 | sym::log @@ -429,6 +409,22 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc (sym::fabs, F32) => self.context.get_builtin_function("fabsf"), (sym::fabs, F64) => self.context.get_builtin_function("fabs"), + (sym::floor, F32) => self.context.get_builtin_function("floorf"), + (sym::floor, F64) => self.context.get_builtin_function("floor"), + + (sym::ceil, F32) => self.context.get_builtin_function("ceilf"), + (sym::ceil, F64) => self.context.get_builtin_function("ceil"), + + (sym::trunc, F32) => self.context.get_builtin_function("truncf"), + (sym::trunc, F64) => self.context.get_builtin_function("trunc"), + + (sym::round, F32) => self.context.get_builtin_function("roundf"), + (sym::round, F64) => self.context.get_builtin_function("round"), + + // We match the LLVM backend and lower this to `rint`. + (sym::round_ties_even, F32) => self.context.get_builtin_function("rintf"), + (sym::round_ties_even, F64) => self.context.get_builtin_function("rint"), + (sym::exp, F32) => self.context.get_builtin_function("expf"), (sym::exp, F64) => self.context.get_builtin_function("exp"), diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index c4ff1eee56750..fa61f54e5c169 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -100,35 +100,6 @@ fn call_simple_intrinsic<'ll, 'tcx>( sym::copysignf64 => ("llvm.copysign", &[bx.type_f64()]), sym::copysignf128 => ("llvm.copysign", &[bx.type_f128()]), - sym::floorf16 => ("llvm.floor", &[bx.type_f16()]), - sym::floorf32 => ("llvm.floor", &[bx.type_f32()]), - sym::floorf64 => ("llvm.floor", &[bx.type_f64()]), - sym::floorf128 => ("llvm.floor", &[bx.type_f128()]), - - sym::ceilf16 => ("llvm.ceil", &[bx.type_f16()]), - sym::ceilf32 => ("llvm.ceil", &[bx.type_f32()]), - sym::ceilf64 => ("llvm.ceil", &[bx.type_f64()]), - sym::ceilf128 => ("llvm.ceil", &[bx.type_f128()]), - - sym::truncf16 => ("llvm.trunc", &[bx.type_f16()]), - sym::truncf32 => ("llvm.trunc", &[bx.type_f32()]), - sym::truncf64 => ("llvm.trunc", &[bx.type_f64()]), - sym::truncf128 => ("llvm.trunc", &[bx.type_f128()]), - - // We could use any of `rint`, `nearbyint`, or `roundeven` - // for this -- they are all identical in semantics when - // assuming the default FP environment. - // `rint` is what we used for $forever. - sym::round_ties_even_f16 => ("llvm.rint", &[bx.type_f16()]), - sym::round_ties_even_f32 => ("llvm.rint", &[bx.type_f32()]), - sym::round_ties_even_f64 => ("llvm.rint", &[bx.type_f64()]), - sym::round_ties_even_f128 => ("llvm.rint", &[bx.type_f128()]), - - sym::roundf16 => ("llvm.round", &[bx.type_f16()]), - sym::roundf32 => ("llvm.round", &[bx.type_f32()]), - sym::roundf64 => ("llvm.round", &[bx.type_f64()]), - sym::roundf128 => ("llvm.round", &[bx.type_f128()]), - _ => return None, }; Some(bx.call_intrinsic( @@ -568,6 +539,11 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } sym::fabs + | sym::floor + | sym::ceil + | sym::trunc + | sym::round + | sym::round_ties_even | sym::exp | sym::exp2 | sym::log @@ -587,6 +563,15 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { let llty = self.type_float_from_ty(*f); let llvm_name = match name { sym::fabs => "llvm.fabs", + sym::floor => "llvm.floor", + sym::ceil => "llvm.ceil", + sym::trunc => "llvm.trunc", + sym::round => "llvm.round", + // We could use any of `rint`, `nearbyint`, or `roundeven` + // for this -- they are all identical in semantics when + // assuming the default FP environment. + // `rint` is what we used for $forever. + sym::round_ties_even => "llvm.rint", sym::exp => "llvm.exp", sym::exp2 => "llvm.exp2", sym::log => "llvm.log", diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 775e1dcf2ffea..6ba4303321479 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -345,10 +345,10 @@ impl CodegenBackend for LlvmCodegenBackend { sym::log, sym::log10, sym::log2, + sym::floor, sym::ceil, sym::trunc, + sym::round, sym::round_ties_even, // Fallback via f32 or f64, but the LLVM intrinsic is used instead. - sym::floorf16, sym::ceilf16, sym::truncf16, - sym::round_ties_even_f16, sym::roundf16, sym::sqrtf16, sym::powif16, sym::fmaf16, diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index 00057dc503827..acf5369fd0156 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -617,7 +617,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { sym::copysignf64 => self.float_copysign_intrinsic::(args, dest)?, sym::copysignf128 => self.float_copysign_intrinsic::(args, dest)?, - sym::fabs => { + sym::fabs | sym::floor | sym::ceil | sym::trunc | sym::round | sym::round_ties_even => { let arg = self.read_immediate(&args[0])?; let ty::Float(float_ty) = arg.layout.ty.kind() else { span_bug!( @@ -635,102 +635,6 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self.write_scalar(out_val, dest)?; } - sym::floorf16 => self.float_round_intrinsic::( - args, - dest, - rustc_apfloat::Round::TowardNegative, - )?, - sym::floorf32 => self.float_round_intrinsic::( - args, - dest, - rustc_apfloat::Round::TowardNegative, - )?, - sym::floorf64 => self.float_round_intrinsic::( - args, - dest, - rustc_apfloat::Round::TowardNegative, - )?, - sym::floorf128 => self.float_round_intrinsic::( - args, - dest, - rustc_apfloat::Round::TowardNegative, - )?, - - sym::ceilf16 => self.float_round_intrinsic::( - args, - dest, - rustc_apfloat::Round::TowardPositive, - )?, - sym::ceilf32 => self.float_round_intrinsic::( - args, - dest, - rustc_apfloat::Round::TowardPositive, - )?, - sym::ceilf64 => self.float_round_intrinsic::( - args, - dest, - rustc_apfloat::Round::TowardPositive, - )?, - sym::ceilf128 => self.float_round_intrinsic::( - args, - dest, - rustc_apfloat::Round::TowardPositive, - )?, - - sym::truncf16 => { - self.float_round_intrinsic::(args, dest, rustc_apfloat::Round::TowardZero)? - } - sym::truncf32 => { - self.float_round_intrinsic::(args, dest, rustc_apfloat::Round::TowardZero)? - } - sym::truncf64 => { - self.float_round_intrinsic::(args, dest, rustc_apfloat::Round::TowardZero)? - } - sym::truncf128 => { - self.float_round_intrinsic::(args, dest, rustc_apfloat::Round::TowardZero)? - } - - sym::roundf16 => self.float_round_intrinsic::( - args, - dest, - rustc_apfloat::Round::NearestTiesToAway, - )?, - sym::roundf32 => self.float_round_intrinsic::( - args, - dest, - rustc_apfloat::Round::NearestTiesToAway, - )?, - sym::roundf64 => self.float_round_intrinsic::( - args, - dest, - rustc_apfloat::Round::NearestTiesToAway, - )?, - sym::roundf128 => self.float_round_intrinsic::( - args, - dest, - rustc_apfloat::Round::NearestTiesToAway, - )?, - - sym::round_ties_even_f16 => self.float_round_intrinsic::( - args, - dest, - rustc_apfloat::Round::NearestTiesToEven, - )?, - sym::round_ties_even_f32 => self.float_round_intrinsic::( - args, - dest, - rustc_apfloat::Round::NearestTiesToEven, - )?, - sym::round_ties_even_f64 => self.float_round_intrinsic::( - args, - dest, - rustc_apfloat::Round::NearestTiesToEven, - )?, - sym::round_ties_even_f128 => self.float_round_intrinsic::( - args, - dest, - rustc_apfloat::Round::NearestTiesToEven, - )?, sym::fmaf16 => self.float_muladd_intrinsic::(args, dest, MulAddType::Fused)?, sym::fmaf32 => self.float_muladd_intrinsic::(args, dest, MulAddType::Fused)?, sym::fmaf64 => self.float_muladd_intrinsic::(args, dest, MulAddType::Fused)?, @@ -1192,7 +1096,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } fn unop_float_intrinsic( - &self, + &mut self, name: Symbol, arg: ImmTy<'tcx, M::Provenance>, ) -> InterpResult<'tcx, Scalar> @@ -1203,6 +1107,13 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { match name { // bitwise, no NaN adjustments sym::fabs => interp_ok(x.abs().into()), + + sym::floor => self.float_round(x, rustc_apfloat::Round::TowardNegative), + sym::ceil => self.float_round(x, rustc_apfloat::Round::TowardPositive), + sym::trunc => self.float_round(x, rustc_apfloat::Round::TowardZero), + sym::round => self.float_round(x, rustc_apfloat::Round::NearestTiesToAway), + sym::round_ties_even => self.float_round(x, rustc_apfloat::Round::NearestTiesToEven), + _ => bug!("not a unary float intrinsic: {}", name), } } @@ -1267,32 +1178,17 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { fn float_round( &mut self, - x: Scalar, + x: F, mode: rustc_apfloat::Round, ) -> InterpResult<'tcx, Scalar> where F: rustc_apfloat::Float + rustc_apfloat::FloatConvert + Into>, { - let x: F = x.to_float()?; let res = x.round_to_integral(mode).value; let res = self.adjust_nan(res, &[x]); interp_ok(res.into()) } - fn float_round_intrinsic( - &mut self, - args: &[OpTy<'tcx, M::Provenance>], - dest: &PlaceTy<'tcx, M::Provenance>, - mode: rustc_apfloat::Round, - ) -> InterpResult<'tcx, ()> - where - F: rustc_apfloat::Float + rustc_apfloat::FloatConvert + Into>, - { - let res = self.float_round::(self.read_scalar(&args[0])?, mode)?; - self.write_scalar(res, dest)?; - interp_ok(()) - } - fn float_muladd( &self, a: Scalar, diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs b/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs index 2ddb20fe8c987..7c5f63eb236ac 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs @@ -145,10 +145,18 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { }; let op = op.to_scalar(); match float_ty { - FloatTy::F16 => self.float_round::(op, rounding)?, - FloatTy::F32 => self.float_round::(op, rounding)?, - FloatTy::F64 => self.float_round::(op, rounding)?, - FloatTy::F128 => self.float_round::(op, rounding)?, + FloatTy::F16 => { + self.float_round::(op.to_float()?, rounding)? + } + FloatTy::F32 => { + self.float_round::(op.to_float()?, rounding)? + } + FloatTy::F64 => { + self.float_round::(op.to_float()?, rounding)? + } + FloatTy::F128 => { + self.float_round::(op.to_float()?, rounding)? + } } } Op::Numeric(name) => { diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index 30d7127ccd8fa..926b2ff24561f 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -82,10 +82,7 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi | sym::caller_location | sym::carrying_mul_add | sym::carryless_mul - | sym::ceilf16 - | sym::ceilf32 - | sym::ceilf64 - | sym::ceilf128 + | sym::ceil | sym::cold_path | sym::const_eval_select | sym::contract_check_ensures @@ -109,10 +106,7 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi | sym::field_representing_type_actual_type_id | sym::field_representing_type_name | sym::field_representing_type_offset - | sym::floorf16 - | sym::floorf32 - | sym::floorf64 - | sym::floorf128 + | sym::floor | sym::fmaf16 | sym::fmaf32 | sym::fmaf64 @@ -173,14 +167,8 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi | sym::return_address | sym::rotate_left | sym::rotate_right - | sym::round_ties_even_f16 - | sym::round_ties_even_f32 - | sym::round_ties_even_f64 - | sym::round_ties_even_f128 - | sym::roundf16 - | sym::roundf32 - | sym::roundf64 - | sym::roundf128 + | sym::round + | sym::round_ties_even | sym::rustc_peek | sym::saturating_add | sym::saturating_sub @@ -194,10 +182,7 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi | sym::sqrtf128 | sym::sub_with_overflow | sym::three_way_compare - | sym::truncf16 - | sym::truncf32 - | sym::truncf64 - | sym::truncf128 + | sym::trunc | sym::type_id | sym::type_id_eq | sym::type_id_field_representing_type @@ -446,7 +431,12 @@ pub(crate) fn check_intrinsic_type( | sym::log10 | sym::fabs | sym::sin - | sym::cos => (1, 0, vec![param(0)], param(0)), + | sym::cos + | sym::floor + | sym::ceil + | sym::trunc + | sym::round_ties_even + | sym::round => (1, 0, vec![param(0)], param(0)), sym::minimum_number_nsz_f16 => (0, 0, vec![tcx.types.f16, tcx.types.f16], tcx.types.f16), sym::minimum_number_nsz_f32 => (0, 0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32), @@ -477,31 +467,6 @@ pub(crate) fn check_intrinsic_type( sym::copysignf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64), sym::copysignf128 => (0, 0, vec![tcx.types.f128, tcx.types.f128], tcx.types.f128), - sym::floorf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16), - sym::floorf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32), - sym::floorf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64), - sym::floorf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128), - - sym::ceilf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16), - sym::ceilf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32), - sym::ceilf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64), - sym::ceilf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128), - - sym::truncf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16), - sym::truncf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32), - sym::truncf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64), - sym::truncf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128), - - sym::round_ties_even_f16 => (0, 0, vec![tcx.types.f16], tcx.types.f16), - sym::round_ties_even_f32 => (0, 0, vec![tcx.types.f32], tcx.types.f32), - sym::round_ties_even_f64 => (0, 0, vec![tcx.types.f64], tcx.types.f64), - sym::round_ties_even_f128 => (0, 0, vec![tcx.types.f128], tcx.types.f128), - - sym::roundf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16), - sym::roundf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32), - sym::roundf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64), - sym::roundf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128), - sym::volatile_load | sym::unaligned_volatile_load => { (1, 0, vec![Ty::new_imm_ptr(tcx, param(0))], param(0)) } diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 7665df4a4e5ae..ea537518e4ebf 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -576,10 +576,7 @@ symbols! { catch_unwind, cause, cdylib, - ceilf16, - ceilf32, - ceilf64, - ceilf128, + ceil, cfg, cfg_accessible, cfg_attr, @@ -974,10 +971,7 @@ symbols! { file, final_associated_functions, float_to_int_unchecked, - floorf16, - floorf32, - floorf64, - floorf128, + floor, fma4_target_feature, fmaf16, fmaf32, @@ -1726,14 +1720,8 @@ symbols! { ropi_rwpi: "ropi-rwpi", rotate_left, rotate_right, - round_ties_even_f16, - round_ties_even_f32, - round_ties_even_f64, - round_ties_even_f128, - roundf16, - roundf32, - roundf64, - roundf128, + round, + round_ties_even, rtm_target_feature, runtime, rust, @@ -2142,10 +2130,7 @@ symbols! { transparent_unions, trivial_bounds, trivial_clone, - truncf16, - truncf32, - truncf64, - truncf128, + trunc, try_as_dyn, try_blocks, try_blocks_heterogeneous, diff --git a/library/core/src/intrinsics/macros.rs b/library/core/src/intrinsics/macros.rs index 5abad0e2c9726..587994c2929d3 100644 --- a/library/core/src/intrinsics/macros.rs +++ b/library/core/src/intrinsics/macros.rs @@ -24,6 +24,41 @@ macro_rules! intrinsic_dispatch_on_type { $($arg:ident: $arg_ty:ty),* $(,)? ) -> $ret_ty:ty; $($concrete:ty => $body:block)* + ) => { + intrinsic_dispatch_on_type! { + @dispatch_mod $name<$generic: $bound>($($arg: $arg_ty),*) -> $ret_ty, + $($concrete => $body)* + } + + $(#[$attr])* + $vis fn $name<$generic: $name::Dispatch>($($arg: $arg_ty),*) -> $ret_ty { + <$generic as $name::Dispatch>::dispatch($($arg),*) + } + }; + + ( + $(#[$attr:meta])* + $vis:vis const fn $name:ident<$generic:ident: $bound:path>( + $($arg:ident: $arg_ty:ty),* $(,)? + ) -> $ret_ty:ty; + $($concrete:ty => $body:block)* + ) => { + intrinsic_dispatch_on_type! { + @dispatch_mod $name<$generic: $bound>($($arg: $arg_ty),*) -> $ret_ty, + $($concrete => $body)* + } + + $(#[$attr])* + $vis const fn $name<$generic: $name::Dispatch>($($arg: $arg_ty),*) -> $ret_ty { + <$generic as $name::Dispatch>::dispatch($($arg),*) + } + }; + + ( + @dispatch_mod $name:ident<$generic:ident: $bound:path>( + $($arg:ident: $arg_ty:ty),* + ) -> $ret_ty:ty, + $($concrete:ty => $body:block)* ) => { mod $name { use super::*; @@ -37,11 +72,6 @@ macro_rules! intrinsic_dispatch_on_type { $($concrete => $body)* } } - - $(#[$attr])* - $vis fn $name<$generic: $name::Dispatch>($($arg: $arg_ty),*) -> $ret_ty { - <$generic as $name::Dispatch>::dispatch($($arg),*) - } }; ( diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index a99633456de0b..b6d16a6a02b54 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -1410,192 +1410,108 @@ pub const fn fmuladdf128(a: f128, b: f128, c: f128) -> f128 { a * b + c } -/// Returns the largest integer less than or equal to an `f16`. -/// -/// The stabilized version of this intrinsic is -/// [`f16::floor`](../../std/primitive.f16.html#method.floor) -#[rustc_intrinsic_const_stable_indirect] -#[inline] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn floorf16(x: f16) -> f16 { - floorf32(x as f32) as f16 -} -/// Returns the largest integer less than or equal to an `f32`. -/// -/// The stabilized version of this intrinsic is -/// [`f32::floor`](../../std/primitive.f32.html#method.floor) -#[rustc_intrinsic_const_stable_indirect] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn floorf32(x: f32) -> f32; -/// Returns the largest integer less than or equal to an `f64`. -/// -/// The stabilized version of this intrinsic is -/// [`f64::floor`](../../std/primitive.f64.html#method.floor) -#[rustc_intrinsic_const_stable_indirect] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn floorf64(x: f64) -> f64; -/// Returns the largest integer less than or equal to an `f128`. -/// -/// The stabilized version of this intrinsic is -/// [`f128::floor`](../../std/primitive.f128.html#method.floor) -#[rustc_intrinsic_const_stable_indirect] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn floorf128(x: f128) -> f128; +intrinsic_dispatch_on_type! { + /// Returns the largest integer less than or equal to a floating-point value. + /// + /// The stabilized versions of this intrinsic are available on the float primitives via the + /// `floor` method. For example, [`f32::floor`](../../std/primitive.f32.html#method.floor). + #[rustc_nounwind] + #[inline] + #[rustc_intrinsic_const_stable_indirect] + #[rustc_intrinsic] + // The fallback bodies call into `libm`, which is not available at compile time. This is fine + // because const-eval implements this intrinsic itself and never runs the fallback body. + #[rustc_do_not_const_check] + pub const fn floor(x: T) -> T; -/// Returns the smallest integer greater than or equal to an `f16`. -/// -/// The stabilized version of this intrinsic is -/// [`f16::ceil`](../../std/primitive.f16.html#method.ceil) -#[rustc_intrinsic_const_stable_indirect] -#[inline] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn ceilf16(x: f16) -> f16 { - ceilf32(x as f32) as f16 + f16 => { libm::floorf16(x) } + f32 => { libm::floorf(x) } + f64 => { libm::floor(x) } + f128 => { libm::floorf128(x) } } -/// Returns the smallest integer greater than or equal to an `f32`. -/// -/// The stabilized version of this intrinsic is -/// [`f32::ceil`](../../std/primitive.f32.html#method.ceil) -#[rustc_intrinsic_const_stable_indirect] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn ceilf32(x: f32) -> f32; -/// Returns the smallest integer greater than or equal to an `f64`. -/// -/// The stabilized version of this intrinsic is -/// [`f64::ceil`](../../std/primitive.f64.html#method.ceil) -#[rustc_intrinsic_const_stable_indirect] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn ceilf64(x: f64) -> f64; -/// Returns the smallest integer greater than or equal to an `f128`. -/// -/// The stabilized version of this intrinsic is -/// [`f128::ceil`](../../std/primitive.f128.html#method.ceil) -#[rustc_intrinsic_const_stable_indirect] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn ceilf128(x: f128) -> f128; -/// Returns the integer part of an `f16`. -/// -/// The stabilized version of this intrinsic is -/// [`f16::trunc`](../../std/primitive.f16.html#method.trunc) -#[rustc_intrinsic_const_stable_indirect] -#[inline] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn truncf16(x: f16) -> f16 { - truncf32(x as f32) as f16 +intrinsic_dispatch_on_type! { + /// Returns the smallest integer greater than or equal to a floating-point value. + /// + /// The stabilized versions of this intrinsic are available on the float primitives via the + /// `ceil` method. For example, [`f32::ceil`](../../std/primitive.f32.html#method.ceil). + #[rustc_nounwind] + #[inline] + #[rustc_intrinsic_const_stable_indirect] + #[rustc_intrinsic] + // The fallback bodies call into `libm`, which is not available at compile time. This is fine + // because const-eval implements this intrinsic itself and never runs the fallback body. + #[rustc_do_not_const_check] + pub const fn ceil(x: T) -> T; + + f16 => { libm::ceilf16(x) } + f32 => { libm::ceilf(x) } + f64 => { libm::ceil(x) } + f128 => { libm::ceilf128(x) } } -/// Returns the integer part of an `f32`. -/// -/// The stabilized version of this intrinsic is -/// [`f32::trunc`](../../std/primitive.f32.html#method.trunc) -#[rustc_intrinsic_const_stable_indirect] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn truncf32(x: f32) -> f32; -/// Returns the integer part of an `f64`. -/// -/// The stabilized version of this intrinsic is -/// [`f64::trunc`](../../std/primitive.f64.html#method.trunc) -#[rustc_intrinsic_const_stable_indirect] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn truncf64(x: f64) -> f64; -/// Returns the integer part of an `f128`. -/// -/// The stabilized version of this intrinsic is -/// [`f128::trunc`](../../std/primitive.f128.html#method.trunc) -#[rustc_intrinsic_const_stable_indirect] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn truncf128(x: f128) -> f128; -/// Returns the nearest integer to an `f16`. Rounds half-way cases to the number with an even -/// least significant digit. -/// -/// The stabilized version of this intrinsic is -/// [`f16::round_ties_even`](../../std/primitive.f16.html#method.round_ties_even) -#[rustc_intrinsic_const_stable_indirect] -#[inline] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn round_ties_even_f16(x: f16) -> f16 { - round_ties_even_f32(x as f32) as f16 +intrinsic_dispatch_on_type! { + /// Returns the integer part of a floating-point value. + /// + /// The stabilized versions of this intrinsic are available on the float primitives via the + /// `trunc` method. For example, [`f32::trunc`](../../std/primitive.f32.html#method.trunc). + #[rustc_nounwind] + #[inline] + #[rustc_intrinsic_const_stable_indirect] + #[rustc_intrinsic] + // The fallback bodies call into `libm`, which is not available at compile time. This is fine + // because const-eval implements this intrinsic itself and never runs the fallback body. + #[rustc_do_not_const_check] + pub const fn trunc(x: T) -> T; + + f16 => { libm::truncf16(x) } + f32 => { libm::truncf(x) } + f64 => { libm::trunc(x) } + f128 => { libm::truncf128(x) } } -/// Returns the nearest integer to an `f32`. Rounds half-way cases to the number with an even -/// least significant digit. -/// -/// The stabilized version of this intrinsic is -/// [`f32::round_ties_even`](../../std/primitive.f32.html#method.round_ties_even) -#[rustc_intrinsic_const_stable_indirect] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn round_ties_even_f32(x: f32) -> f32; +intrinsic_dispatch_on_type! { + /// Returns the nearest integer to a floating-point value. Rounds half-way cases to the number + /// with an even least significant digit. + /// + /// The stabilized versions of this intrinsic are available on the float primitives via the + /// `round_ties_even` method. For example, + /// [`f32::round_ties_even`](../../std/primitive.f32.html#method.round_ties_even). + #[rustc_nounwind] + #[inline] + #[rustc_intrinsic_const_stable_indirect] + #[rustc_intrinsic] + // The fallback bodies call into `libm`, which is not available at compile time. This is fine + // because const-eval implements this intrinsic itself and never runs the fallback body. + #[rustc_do_not_const_check] + pub const fn round_ties_even(x: T) -> T; -/// Returns the nearest integer to an `f64`. Rounds half-way cases to the number with an even -/// least significant digit. -/// -/// The stabilized version of this intrinsic is -/// [`f64::round_ties_even`](../../std/primitive.f64.html#method.round_ties_even) -#[rustc_intrinsic_const_stable_indirect] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn round_ties_even_f64(x: f64) -> f64; + f16 => { libm::roundevenf16(x) } + f32 => { libm::roundevenf(x) } + f64 => { libm::roundeven(x) } + f128 => { libm::roundevenf128(x) } +} -/// Returns the nearest integer to an `f128`. Rounds half-way cases to the number with an even -/// least significant digit. -/// -/// The stabilized version of this intrinsic is -/// [`f128::round_ties_even`](../../std/primitive.f128.html#method.round_ties_even) -#[rustc_intrinsic_const_stable_indirect] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn round_ties_even_f128(x: f128) -> f128; +intrinsic_dispatch_on_type! { + /// Returns the nearest integer to a floating-point value. Rounds half-way cases away from + /// zero. + /// + /// The stabilized versions of this intrinsic are available on the float primitives via the + /// `round` method. For example, [`f32::round`](../../std/primitive.f32.html#method.round). + #[rustc_nounwind] + #[inline] + #[rustc_intrinsic_const_stable_indirect] + #[rustc_intrinsic] + // The fallback bodies call into `libm`, which is not available at compile time. This is fine + // because const-eval implements this intrinsic itself and never runs the fallback body. + #[rustc_do_not_const_check] + pub const fn round(x: T) -> T; -/// Returns the nearest integer to an `f16`. Rounds half-way cases away from zero. -/// -/// The stabilized version of this intrinsic is -/// [`f16::round`](../../std/primitive.f16.html#method.round) -#[rustc_intrinsic_const_stable_indirect] -#[inline] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn roundf16(x: f16) -> f16 { - roundf32(x as f32) as f16 + f16 => { libm::roundf16(x) } + f32 => { libm::roundf(x) } + f64 => { libm::round(x) } + f128 => { libm::roundf128(x) } } -/// Returns the nearest integer to an `f32`. Rounds half-way cases away from zero. -/// -/// The stabilized version of this intrinsic is -/// [`f32::round`](../../std/primitive.f32.html#method.round) -#[rustc_intrinsic_const_stable_indirect] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn roundf32(x: f32) -> f32; -/// Returns the nearest integer to an `f64`. Rounds half-way cases away from zero. -/// -/// The stabilized version of this intrinsic is -/// [`f64::round`](../../std/primitive.f64.html#method.round) -#[rustc_intrinsic_const_stable_indirect] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn roundf64(x: f64) -> f64; -/// Returns the nearest integer to an `f128`. Rounds half-way cases away from zero. -/// -/// The stabilized version of this intrinsic is -/// [`f128::round`](../../std/primitive.f128.html#method.round) -#[rustc_intrinsic_const_stable_indirect] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn roundf128(x: f128) -> f128; /// Float addition that allows optimizations based on algebraic rules. /// Requires that inputs and output of the operation are finite, causing UB otherwise. diff --git a/library/core/src/num/f128.rs b/library/core/src/num/f128.rs index 93dd52198f488..db69c87bd59af 100644 --- a/library/core/src/num/f128.rs +++ b/library/core/src/num/f128.rs @@ -1799,7 +1799,7 @@ impl f128 { #[rustc_const_unstable(feature = "f128", issue = "116909")] #[must_use = "method returns a new number and does not mutate the original value"] pub const fn floor(self) -> f128 { - intrinsics::floorf128(self) + intrinsics::floor(self) } /// Returns the smallest integer greater than or equal to `self`. @@ -1826,7 +1826,7 @@ impl f128 { #[rustc_const_unstable(feature = "f128", issue = "116909")] #[must_use = "method returns a new number and does not mutate the original value"] pub const fn ceil(self) -> f128 { - intrinsics::ceilf128(self) + intrinsics::ceil(self) } /// Returns the nearest integer to `self`. If a value is half-way between two @@ -1859,7 +1859,7 @@ impl f128 { #[rustc_const_unstable(feature = "f128", issue = "116909")] #[must_use = "method returns a new number and does not mutate the original value"] pub const fn round(self) -> f128 { - intrinsics::roundf128(self) + intrinsics::round(self) } /// Returns the nearest integer to a number. Rounds half-way cases to the number @@ -1890,7 +1890,7 @@ impl f128 { #[rustc_const_unstable(feature = "f128", issue = "116909")] #[must_use = "method returns a new number and does not mutate the original value"] pub const fn round_ties_even(self) -> f128 { - intrinsics::round_ties_even_f128(self) + intrinsics::round_ties_even(self) } /// Returns the integer part of `self`. @@ -1920,7 +1920,7 @@ impl f128 { #[rustc_const_unstable(feature = "f128", issue = "116909")] #[must_use = "method returns a new number and does not mutate the original value"] pub const fn trunc(self) -> f128 { - intrinsics::truncf128(self) + intrinsics::trunc(self) } /// Returns the fractional part of `self`. diff --git a/library/core/src/num/f16.rs b/library/core/src/num/f16.rs index cb79c0736c608..60a4edeeebdb5 100644 --- a/library/core/src/num/f16.rs +++ b/library/core/src/num/f16.rs @@ -1785,7 +1785,7 @@ impl f16 { #[rustc_const_unstable(feature = "f16", issue = "116909")] #[must_use = "method returns a new number and does not mutate the original value"] pub const fn floor(self) -> f16 { - intrinsics::floorf16(self) + intrinsics::floor(self) } /// Returns the smallest integer greater than or equal to `self`. @@ -1812,7 +1812,7 @@ impl f16 { #[rustc_const_unstable(feature = "f16", issue = "116909")] #[must_use = "method returns a new number and does not mutate the original value"] pub const fn ceil(self) -> f16 { - intrinsics::ceilf16(self) + intrinsics::ceil(self) } /// Returns the nearest integer to `self`. If a value is half-way between two @@ -1845,7 +1845,7 @@ impl f16 { #[rustc_const_unstable(feature = "f16", issue = "116909")] #[must_use = "method returns a new number and does not mutate the original value"] pub const fn round(self) -> f16 { - intrinsics::roundf16(self) + intrinsics::round(self) } /// Returns the nearest integer to a number. Rounds half-way cases to the number @@ -1876,7 +1876,7 @@ impl f16 { #[rustc_const_unstable(feature = "f16", issue = "116909")] #[must_use = "method returns a new number and does not mutate the original value"] pub const fn round_ties_even(self) -> f16 { - intrinsics::round_ties_even_f16(self) + intrinsics::round_ties_even(self) } /// Returns the integer part of `self`. @@ -1906,7 +1906,7 @@ impl f16 { #[rustc_const_unstable(feature = "f16", issue = "116909")] #[must_use = "method returns a new number and does not mutate the original value"] pub const fn trunc(self) -> f16 { - intrinsics::truncf16(self) + intrinsics::trunc(self) } /// Returns the fractional part of `self`. diff --git a/library/core/src/num/f32.rs b/library/core/src/num/f32.rs index 8a02aa7517474..3c22f05020cce 100644 --- a/library/core/src/num/f32.rs +++ b/library/core/src/num/f32.rs @@ -1940,7 +1940,7 @@ pub mod math { #[unstable(feature = "core_float_math", issue = "137578")] #[must_use = "method returns a new number and does not mutate the original value"] pub const fn floor(x: f32) -> f32 { - intrinsics::floorf32(x) + intrinsics::floor(x) } /// Experimental version of `ceil` in `core`. See [`f32::ceil`] for details. @@ -1968,7 +1968,7 @@ pub mod math { #[must_use = "method returns a new number and does not mutate the original value"] #[unstable(feature = "core_float_math", issue = "137578")] pub const fn ceil(x: f32) -> f32 { - intrinsics::ceilf32(x) + intrinsics::ceil(x) } /// Experimental version of `round` in `core`. See [`f32::round`] for details. @@ -2001,7 +2001,7 @@ pub mod math { #[unstable(feature = "core_float_math", issue = "137578")] #[must_use = "method returns a new number and does not mutate the original value"] pub const fn round(x: f32) -> f32 { - intrinsics::roundf32(x) + intrinsics::round(x) } /// Experimental version of `round_ties_even` in `core`. See [`f32::round_ties_even`] for @@ -2033,7 +2033,7 @@ pub mod math { #[unstable(feature = "core_float_math", issue = "137578")] #[must_use = "method returns a new number and does not mutate the original value"] pub const fn round_ties_even(x: f32) -> f32 { - intrinsics::round_ties_even_f32(x) + intrinsics::round_ties_even(x) } /// Experimental version of `trunc` in `core`. See [`f32::trunc`] for details. @@ -2063,7 +2063,7 @@ pub mod math { #[must_use = "method returns a new number and does not mutate the original value"] #[unstable(feature = "core_float_math", issue = "137578")] pub const fn trunc(x: f32) -> f32 { - intrinsics::truncf32(x) + intrinsics::trunc(x) } /// Experimental version of `fract` in `core`. See [`f32::fract`] for details. diff --git a/library/core/src/num/f64.rs b/library/core/src/num/f64.rs index e0bb0e35415b6..215e38e84748e 100644 --- a/library/core/src/num/f64.rs +++ b/library/core/src/num/f64.rs @@ -1918,7 +1918,7 @@ pub mod math { #[unstable(feature = "core_float_math", issue = "137578")] #[must_use = "method returns a new number and does not mutate the original value"] pub const fn floor(x: f64) -> f64 { - intrinsics::floorf64(x) + intrinsics::floor(x) } /// Experimental version of `ceil` in `core`. See [`f64::ceil`] for details. @@ -1946,7 +1946,7 @@ pub mod math { #[unstable(feature = "core_float_math", issue = "137578")] #[must_use = "method returns a new number and does not mutate the original value"] pub const fn ceil(x: f64) -> f64 { - intrinsics::ceilf64(x) + intrinsics::ceil(x) } /// Experimental version of `round` in `core`. See [`f64::round`] for details. @@ -1979,7 +1979,7 @@ pub mod math { #[unstable(feature = "core_float_math", issue = "137578")] #[must_use = "method returns a new number and does not mutate the original value"] pub const fn round(x: f64) -> f64 { - intrinsics::roundf64(x) + intrinsics::round(x) } /// Experimental version of `round_ties_even` in `core`. See [`f64::round_ties_even`] for @@ -2011,7 +2011,7 @@ pub mod math { #[unstable(feature = "core_float_math", issue = "137578")] #[must_use = "method returns a new number and does not mutate the original value"] pub const fn round_ties_even(x: f64) -> f64 { - intrinsics::round_ties_even_f64(x) + intrinsics::round_ties_even(x) } /// Experimental version of `trunc` in `core`. See [`f64::trunc`] for details. @@ -2041,7 +2041,7 @@ pub mod math { #[unstable(feature = "core_float_math", issue = "137578")] #[must_use = "method returns a new number and does not mutate the original value"] pub const fn trunc(x: f64) -> f64 { - intrinsics::truncf64(x) + intrinsics::trunc(x) } /// Experimental version of `fract` in `core`. See [`f64::fract`] for details. diff --git a/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs b/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs index 9fb0e1464683e..2489899477f96 100644 --- a/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs +++ b/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs @@ -22160,7 +22160,7 @@ pub fn vrndaq_f64(a: float64x2_t) -> float64x2_t { #[cfg(not(target_arch = "arm64ec"))] #[cfg_attr(test, assert_instr(frinta))] pub fn vrndah_f16(a: f16) -> f16 { - roundf16(a) + round(a) } #[doc = "Floating-point round to integral, to nearest with ties to away"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndh_f16)"] @@ -22170,7 +22170,7 @@ pub fn vrndah_f16(a: f16) -> f16 { #[cfg(not(target_arch = "arm64ec"))] #[cfg_attr(test, assert_instr(frintz))] pub fn vrndh_f16(a: f16) -> f16 { - truncf16(a) + trunc(a) } #[doc = "Floating-point round to integral, using current rounding mode"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndi_f16)"] @@ -22351,7 +22351,7 @@ pub fn vrndmq_f64(a: float64x2_t) -> float64x2_t { #[cfg(not(target_arch = "arm64ec"))] #[cfg_attr(test, assert_instr(frintm))] pub fn vrndmh_f16(a: f16) -> f16 { - floorf16(a) + floor(a) } #[doc = "Floating-point round to integral, to nearest with ties to even"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndn_f64)"] @@ -22482,7 +22482,7 @@ pub fn vrndpq_f64(a: float64x2_t) -> float64x2_t { #[cfg(not(target_arch = "arm64ec"))] #[cfg_attr(test, assert_instr(frintp))] pub fn vrndph_f16(a: f16) -> f16 { - ceilf16(a) + ceil(a) } #[doc = "Floating-point round to integral exact, using current rounding mode"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrndx_f16)"] @@ -22548,7 +22548,7 @@ pub fn vrndxq_f64(a: float64x2_t) -> float64x2_t { #[cfg(not(target_arch = "arm64ec"))] #[cfg_attr(test, assert_instr(frintx))] pub fn vrndxh_f16(a: f16) -> f16 { - round_ties_even_f16(a) + round_ties_even(a) } #[doc = "Signed rounding shift left"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vrshld_s64)"] diff --git a/library/stdarch/crates/core_arch/src/wasm32/mod.rs b/library/stdarch/crates/core_arch/src/wasm32/mod.rs index 57c9157bede89..48d05918208c7 100644 --- a/library/stdarch/crates/core_arch/src/wasm32/mod.rs +++ b/library/stdarch/crates/core_arch/src/wasm32/mod.rs @@ -43,7 +43,7 @@ pub fn unreachable() -> ! { #[must_use = "method returns a new number and does not mutate the original value"] #[unstable(feature = "wasm_numeric_instr", issue = "133908")] pub fn f32_ceil(a: f32) -> f32 { - crate::intrinsics::ceilf32(a) + crate::intrinsics::ceil(a) } /// Generates the [`f32.floor`] instruction, returning the largest integer less than or equal to `a`. @@ -57,7 +57,7 @@ pub fn f32_ceil(a: f32) -> f32 { #[must_use = "method returns a new number and does not mutate the original value"] #[unstable(feature = "wasm_numeric_instr", issue = "133908")] pub fn f32_floor(a: f32) -> f32 { - crate::intrinsics::floorf32(a) + crate::intrinsics::floor(a) } /// Generates the [`f32.trunc`] instruction, roundinging to the nearest integer towards zero. @@ -71,7 +71,7 @@ pub fn f32_floor(a: f32) -> f32 { #[must_use = "method returns a new number and does not mutate the original value"] #[unstable(feature = "wasm_numeric_instr", issue = "133908")] pub fn f32_trunc(a: f32) -> f32 { - crate::intrinsics::truncf32(a) + crate::intrinsics::trunc(a) } /// Generates the [`f32.nearest`] instruction, roundinging to the nearest integer. Rounds half-way @@ -86,7 +86,7 @@ pub fn f32_trunc(a: f32) -> f32 { #[must_use = "method returns a new number and does not mutate the original value"] #[unstable(feature = "wasm_numeric_instr", issue = "133908")] pub fn f32_nearest(a: f32) -> f32 { - crate::intrinsics::round_ties_even_f32(a) + crate::intrinsics::round_ties_even(a) } /// Generates the [`f32.sqrt`] instruction, returning the square root of the number `a`. @@ -114,7 +114,7 @@ pub fn f32_sqrt(a: f32) -> f32 { #[must_use = "method returns a new number and does not mutate the original value"] #[unstable(feature = "wasm_numeric_instr", issue = "133908")] pub fn f64_ceil(a: f64) -> f64 { - crate::intrinsics::ceilf64(a) + crate::intrinsics::ceil(a) } /// Generates the [`f64.floor`] instruction, returning the largest integer less than or equal to `a`. @@ -128,7 +128,7 @@ pub fn f64_ceil(a: f64) -> f64 { #[must_use = "method returns a new number and does not mutate the original value"] #[unstable(feature = "wasm_numeric_instr", issue = "133908")] pub fn f64_floor(a: f64) -> f64 { - crate::intrinsics::floorf64(a) + crate::intrinsics::floor(a) } /// Generates the [`f64.trunc`] instruction, roundinging to the nearest integer towards zero. @@ -142,7 +142,7 @@ pub fn f64_floor(a: f64) -> f64 { #[must_use = "method returns a new number and does not mutate the original value"] #[unstable(feature = "wasm_numeric_instr", issue = "133908")] pub fn f64_trunc(a: f64) -> f64 { - crate::intrinsics::truncf64(a) + crate::intrinsics::trunc(a) } /// Generates the [`f64.nearest`] instruction, roundinging to the nearest integer. Rounds half-way @@ -157,7 +157,7 @@ pub fn f64_trunc(a: f64) -> f64 { #[must_use = "method returns a new number and does not mutate the original value"] #[unstable(feature = "wasm_numeric_instr", issue = "133908")] pub fn f64_nearest(a: f64) -> f64 { - crate::intrinsics::round_ties_even_f64(a) + crate::intrinsics::round_ties_even(a) } /// Generates the [`f64.sqrt`] instruction, returning the square root of the number `a`. diff --git a/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml b/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml index e5ce77ed8b33f..1558be0e373b0 100644 --- a/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml +++ b/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml @@ -2871,7 +2871,7 @@ intrinsics: types: - [f16, 'h_'] compose: - - FnCall: [round_ties_even_f16, [a]] + - FnCall: [round_ties_even, [a]] - name: "vrnda{neon_type.no}" @@ -2920,7 +2920,7 @@ intrinsics: types: - [f16, 'h_'] compose: - - FnCall: [roundf16, [a], []] + - FnCall: [round, [a], []] - name: "vrndn{neon_type.no}" doc: "Floating-point round to integral, to nearest with ties to even" @@ -3020,7 +3020,7 @@ intrinsics: types: - [f16, 'h_'] compose: - - FnCall: [floorf16, [a], []] + - FnCall: [floor, [a], []] @@ -3069,7 +3069,7 @@ intrinsics: types: - [f16, 'h_'] compose: - - FnCall: [ceilf16, [a], []] + - FnCall: [ceil, [a], []] - name: "vrnd{neon_type.no}" doc: "Floating-point round to integral, toward zero" @@ -3116,7 +3116,7 @@ intrinsics: types: - [f16, 'h_'] compose: - - FnCall: [truncf16, [a], []] + - FnCall: [trunc, [a], []] - name: "vrndi{neon_type.no}" diff --git a/src/tools/clippy/clippy_utils/src/sym.rs b/src/tools/clippy/clippy_utils/src/sym.rs index 7d6bb117bd39a..801a68d87dfca 100644 --- a/src/tools/clippy/clippy_utils/src/sym.rs +++ b/src/tools/clippy/clippy_utils/src/sym.rs @@ -161,7 +161,6 @@ generate! { cast, cast_const, cast_mut, - ceil, ceil_char_boundary, chain, char_is_ascii, @@ -273,7 +272,6 @@ generate! { first, flat_map, flatten, - floor, floor_char_boundary, fold, for_each, @@ -518,7 +516,6 @@ generate! { rfind, rmatch_indices, rmatches, - round, rposition, rsplit, rsplit_once, diff --git a/tests/ui/intrinsics/reify-intrinsic.rs b/tests/ui/intrinsics/reify-intrinsic.rs index 8d6f2ea93b723..ef8e02042739f 100644 --- a/tests/ui/intrinsics/reify-intrinsic.rs +++ b/tests/ui/intrinsics/reify-intrinsic.rs @@ -14,7 +14,7 @@ fn b() { fn c() { let _: [unsafe fn(f32) -> f32; 2] = [ - std::intrinsics::floorf32, //~ ERROR cannot coerce + std::intrinsics::floor, //~ ERROR cannot coerce std::intrinsics::log2, ]; } diff --git a/tests/ui/intrinsics/reify-intrinsic.stderr b/tests/ui/intrinsics/reify-intrinsic.stderr index 1307a85c8b6ca..a3904b6280c3f 100644 --- a/tests/ui/intrinsics/reify-intrinsic.stderr +++ b/tests/ui/intrinsics/reify-intrinsic.stderr @@ -18,11 +18,11 @@ LL | let _ = std::mem::transmute as unsafe fn(isize) -> usize; error[E0308]: cannot coerce intrinsics to function pointers --> $DIR/reify-intrinsic.rs:17:9 | -LL | std::intrinsics::floorf32, - | ^^^^^^^^^^^^^^^^^^^^^^^^^ cannot coerce intrinsics to function pointers +LL | std::intrinsics::floor, + | ^^^^^^^^^^^^^^^^^^^^^^ cannot coerce intrinsics to function pointers | - = note: expected fn pointer `unsafe fn(_) -> _` - found fn item `fn(_) -> _ {floorf32}` + = note: expected fn pointer `unsafe fn(f32) -> f32` + found fn item `fn(_) -> _ {std::intrinsics::floor::<_>}` error: aborting due to 3 previous errors From c9d0c7a3d80324ab18ab203fc39ed1ece89efc13 Mon Sep 17 00:00:00 2001 From: N1ark Date: Sun, 6 Sep 2026 19:36:42 +0100 Subject: [PATCH 3/8] Make sqrt generic --- .../src/intrinsics/mod.rs | 10 ++--- .../rustc_codegen_gcc/src/intrinsic/mod.rs | 20 +++------ compiler/rustc_codegen_llvm/src/intrinsic.rs | 7 +-- compiler/rustc_codegen_llvm/src/lib.rs | 3 +- .../rustc_hir_analysis/src/check/intrinsic.rs | 13 ++---- compiler/rustc_span/src/symbol.rs | 5 +-- library/core/src/intrinsics/mod.rs | 43 ++++++------------- library/core/src/num/f128.rs | 2 +- library/core/src/num/f16.rs | 2 +- library/core/src/num/f32.rs | 2 +- library/core/src/num/f64.rs | 2 +- .../core_arch/src/aarch64/neon/generated.rs | 2 +- .../crates/core_arch/src/wasm32/mod.rs | 4 +- .../stdarch/crates/core_arch/src/x86/sse.rs | 4 +- .../stdarch/crates/core_arch/src/x86/sse2.rs | 4 +- .../spec/neon/aarch64.spec.yml | 2 +- src/tools/clippy/clippy_utils/src/sym.rs | 1 - src/tools/miri/src/intrinsics/math.rs | 15 +++++-- .../codegen-llvm/intrinsic-no-unnamed-attr.rs | 4 +- 19 files changed, 57 insertions(+), 88 deletions(-) diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index 3fd8b619f820b..cd2b5de9d07e1 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -322,11 +322,6 @@ fn codegen_float_intrinsic_call<'tcx>( ret: CPlace<'tcx>, ) -> bool { let (name, arg_count, ty, clif_ty) = match intrinsic { - sym::sqrtf16 => return false, // has a fallback via f32 - sym::sqrtf32 => ("sqrtf", 1, fx.tcx.types.f32, types::F32), - sym::sqrtf64 => ("sqrt", 1, fx.tcx.types.f64, types::F64), - sym::sqrtf128 => ("sqrtf128", 1, fx.tcx.types.f128, types::F128), - sym::powif16 => ("__powisf2", 2, fx.tcx.types.f16, types::F16), // compiler-builtins sym::powif32 => ("__powisf2", 2, fx.tcx.types.f32, types::F32), // compiler-builtins sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64, types::F64), // compiler-builtins @@ -393,7 +388,6 @@ fn codegen_float_intrinsic_call<'tcx>( fx.bcx.ins().fma(args[0], args[1], args[2]) } sym::copysignf32 | sym::copysignf64 => fx.bcx.ins().fcopysign(args[0], args[1]), - sym::sqrtf32 | sym::sqrtf64 => fx.bcx.ins().sqrt(args[0]), // These intrinsics aren't supported natively by Cranelift. // Lower them to a libcall. @@ -1142,6 +1136,7 @@ fn codegen_regular_intrinsic_call<'tcx>( } sym::fabs + | sym::sqrt | sym::floor | sym::ceil | sym::trunc @@ -1170,6 +1165,9 @@ fn codegen_regular_intrinsic_call<'tcx>( let res = match (intrinsic, float_ty) { (sym::fabs, F32 | F64) => Codegen(fx.bcx.ins().fabs(x)), + (sym::sqrt, F32 | F64) => Codegen(fx.bcx.ins().sqrt(x)), + (sym::sqrt, F128) => Fallback("sqrtf128"), + (sym::floor, F32 | F64) => Codegen(fx.bcx.ins().floor(x)), (sym::floor, F128) => Fallback("floorf128"), diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index abff9de8575f0..fae22c963283f 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -58,8 +58,6 @@ fn get_simple_intrinsic<'gcc, 'tcx>( name: Symbol, ) -> Option> { let gcc_name = match name { - sym::sqrtf32 => "sqrtf", - sym::sqrtf64 => "sqrt", sym::powif32 => "__builtin_powif", sym::powif64 => "__builtin_powi", sym::powf32 => "powf", @@ -104,7 +102,7 @@ fn get_simple_function_f128<'gcc, 'tcx>( sym::round => "roundf128", sym::round_ties_even => "roundevenf128", sym::sin => "sinf128", - sym::sqrtf128 => "sqrtf128", + sym::sqrt => "sqrtf128", _ => span_bug!(span, "used get_simple_function_f128 for non-unary f128 intrinsic"), }; cx.context.new_function( @@ -138,7 +136,7 @@ fn f16_builtin<'gcc, 'tcx>( sym::round => "__builtin_roundf", sym::round_ties_even => "__builtin_rintf", sym::sin => "sinf", - sym::sqrtf16 => "__builtin_sqrtf", + sym::sqrt => "__builtin_sqrtf", sym::trunc => "__builtin_truncf", _ => unreachable!(), }; @@ -204,15 +202,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc &args.iter().map(|arg| arg.immediate()).collect::>(), ) } - sym::copysignf16 | sym::powf16 | sym::sqrtf16 => f16_builtin(self, name, args), - sym::sqrtf128 if self.cx.supports_f128_type => { - let func = get_simple_function_f128(span, self, name); - self.cx.context.new_call( - self.location, - func, - &args.iter().map(|arg| arg.immediate()).collect::>(), - ) - } + sym::copysignf16 | sym::powf16 => f16_builtin(self, name, args), sym::copysignf128 if self.cx.supports_f128_type => { let f128_type = self.cx.type_f128(); let func = self.cx.context.new_function( @@ -388,6 +378,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc } } sym::fabs + | sym::sqrt | sym::floor | sym::ceil | sym::trunc @@ -409,6 +400,9 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc (sym::fabs, F32) => self.context.get_builtin_function("fabsf"), (sym::fabs, F64) => self.context.get_builtin_function("fabs"), + (sym::sqrt, F32) => self.context.get_builtin_function("sqrtf"), + (sym::sqrt, F64) => self.context.get_builtin_function("sqrt"), + (sym::floor, F32) => self.context.get_builtin_function("floorf"), (sym::floor, F64) => self.context.get_builtin_function("floor"), diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index fa61f54e5c169..71a4c58a59ccd 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -56,11 +56,6 @@ fn call_simple_intrinsic<'ll, 'tcx>( args: &[OperandRef<'tcx, &'ll Value>], ) -> Option<&'ll Value> { let (base_name, type_params): (&'static str, &[&'ll Type]) = match name { - sym::sqrtf16 => ("llvm.sqrt", &[bx.type_f16()]), - sym::sqrtf32 => ("llvm.sqrt", &[bx.type_f32()]), - sym::sqrtf64 => ("llvm.sqrt", &[bx.type_f64()]), - sym::sqrtf128 => ("llvm.sqrt", &[bx.type_f128()]), - sym::powif16 => ("llvm.powi", &[bx.type_f16(), bx.type_i32()]), sym::powif32 => ("llvm.powi", &[bx.type_f32(), bx.type_i32()]), sym::powif64 => ("llvm.powi", &[bx.type_f64(), bx.type_i32()]), @@ -539,6 +534,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } sym::fabs + | sym::sqrt | sym::floor | sym::ceil | sym::trunc @@ -563,6 +559,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { let llty = self.type_float_from_ty(*f); let llvm_name = match name { sym::fabs => "llvm.fabs", + sym::sqrt => "llvm.sqrt", sym::floor => "llvm.floor", sym::ceil => "llvm.ceil", sym::trunc => "llvm.trunc", diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 6ba4303321479..7477d949cfd8c 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -340,6 +340,7 @@ impl CodegenBackend for LlvmCodegenBackend { sym::sin, sym::cos, sym::powf16, sym::powf32, sym::powf64, + sym::sqrt, sym::exp, sym::exp2, sym::log, @@ -349,7 +350,7 @@ impl CodegenBackend for LlvmCodegenBackend { sym::round, sym::round_ties_even, // Fallback via f32 or f64, but the LLVM intrinsic is used instead. - sym::sqrtf16, sym::powif16, + sym::powif16, sym::fmaf16, sym::copysignf16, sym::copysignf32, sym::copysignf64, sym::copysignf128, diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index 926b2ff24561f..5ff81024d0fcb 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -176,10 +176,7 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi | sym::sin | sym::size_of | sym::size_of_type_id - | sym::sqrtf16 - | sym::sqrtf32 - | sym::sqrtf64 - | sym::sqrtf128 + | sym::sqrt | sym::sub_with_overflow | sym::three_way_compare | sym::trunc @@ -395,11 +392,6 @@ pub(crate) fn check_intrinsic_type( tcx.types.unit, ), - sym::sqrtf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16), - sym::sqrtf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32), - sym::sqrtf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64), - sym::sqrtf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128), - sym::powif16 => (0, 0, vec![tcx.types.f16, tcx.types.i32], tcx.types.f16), sym::powif32 => (0, 0, vec![tcx.types.f32, tcx.types.i32], tcx.types.f32), sym::powif64 => (0, 0, vec![tcx.types.f64, tcx.types.i32], tcx.types.f64), @@ -436,7 +428,8 @@ pub(crate) fn check_intrinsic_type( | sym::ceil | sym::trunc | sym::round_ties_even - | sym::round => (1, 0, vec![param(0)], param(0)), + | sym::round + | sym::sqrt => (1, 0, vec![param(0)], param(0)), sym::minimum_number_nsz_f16 => (0, 0, vec![tcx.types.f16, tcx.types.f16], tcx.types.f16), sym::minimum_number_nsz_f32 => (0, 0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32), diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index ea537518e4ebf..45d26cf0e1c33 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -2006,10 +2006,7 @@ symbols! { splat, splatted_index, spotlight, - sqrtf16, - sqrtf32, - sqrtf64, - sqrtf128, + sqrt, sreg, sreg_low16, sse, diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index b6d16a6a02b54..96ad8feb2d74f 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -1052,37 +1052,20 @@ pub unsafe fn unaligned_volatile_load(src: *const T) -> T; #[rustc_diagnostic_item = "intrinsics_unaligned_volatile_store"] pub unsafe fn unaligned_volatile_store(dst: *mut T, val: T); -/// Returns the square root of an `f16` -/// -/// The stabilized version of this intrinsic is -/// [`f16::sqrt`](../../std/primitive.f16.html#method.sqrt) -#[inline] -#[rustc_intrinsic] -#[rustc_nounwind] -pub fn sqrtf16(x: f16) -> f16 { - sqrtf32(x as f32) as f16 +intrinsic_dispatch_on_type! { + /// Returns the square root of a floating-point value. + /// + /// The stabilized versions of this intrinsic are available on the float primitives via the + /// `sqrt` method. For example, [`f32::sqrt`](../../std/primitive.f32.html#method.sqrt). + #[rustc_nounwind] + #[rustc_intrinsic] + pub fn sqrt(x: T) -> T; + + f16 => { libm::sqrtf16(x) } + f32 => { libm::sqrtf(x) } + f64 => { libm::sqrt(x) } + f128 => { libm::sqrtf128(x) } } -/// Returns the square root of an `f32` -/// -/// The stabilized version of this intrinsic is -/// [`f32::sqrt`](../../std/primitive.f32.html#method.sqrt) -#[rustc_intrinsic] -#[rustc_nounwind] -pub fn sqrtf32(x: f32) -> f32; -/// Returns the square root of an `f64` -/// -/// The stabilized version of this intrinsic is -/// [`f64::sqrt`](../../std/primitive.f64.html#method.sqrt) -#[rustc_intrinsic] -#[rustc_nounwind] -pub fn sqrtf64(x: f64) -> f64; -/// Returns the square root of an `f128` -/// -/// The stabilized version of this intrinsic is -/// [`f128::sqrt`](../../std/primitive.f128.html#method.sqrt) -#[rustc_intrinsic] -#[rustc_nounwind] -pub fn sqrtf128(x: f128) -> f128; /// Raises an `f16` to an integer power. /// diff --git a/library/core/src/num/f128.rs b/library/core/src/num/f128.rs index db69c87bd59af..00f4e959064c8 100644 --- a/library/core/src/num/f128.rs +++ b/library/core/src/num/f128.rs @@ -2147,6 +2147,6 @@ impl f128 { #[unstable(feature = "f128", issue = "116909")] #[must_use = "method returns a new number and does not mutate the original value"] pub fn sqrt(self) -> f128 { - intrinsics::sqrtf128(self) + intrinsics::sqrt(self) } } diff --git a/library/core/src/num/f16.rs b/library/core/src/num/f16.rs index 60a4edeeebdb5..e05ff63293af9 100644 --- a/library/core/src/num/f16.rs +++ b/library/core/src/num/f16.rs @@ -2133,7 +2133,7 @@ impl f16 { #[unstable(feature = "f16", issue = "116909")] #[must_use = "method returns a new number and does not mutate the original value"] pub fn sqrt(self) -> f16 { - intrinsics::sqrtf16(self) + intrinsics::sqrt(self) } /// Returns the cube root of a number. diff --git a/library/core/src/num/f32.rs b/library/core/src/num/f32.rs index 3c22f05020cce..d9718bb381365 100644 --- a/library/core/src/num/f32.rs +++ b/library/core/src/num/f32.rs @@ -2259,7 +2259,7 @@ pub mod math { #[unstable(feature = "core_float_math", issue = "137578")] #[must_use = "method returns a new number and does not mutate the original value"] pub fn sqrt(x: f32) -> f32 { - intrinsics::sqrtf32(x) + intrinsics::sqrt(x) } /// Experimental version of `abs_sub` in `core`. See [`f32::abs_sub`] for details. diff --git a/library/core/src/num/f64.rs b/library/core/src/num/f64.rs index 215e38e84748e..ac80eeb7f9d92 100644 --- a/library/core/src/num/f64.rs +++ b/library/core/src/num/f64.rs @@ -2237,7 +2237,7 @@ pub mod math { #[unstable(feature = "core_float_math", issue = "137578")] #[must_use = "method returns a new number and does not mutate the original value"] pub fn sqrt(x: f64) -> f64 { - intrinsics::sqrtf64(x) + intrinsics::sqrt(x) } /// Experimental version of `abs_sub` in `core`. See [`f64::abs_sub`] for details. diff --git a/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs b/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs index 2489899477f96..0fdd332933c2c 100644 --- a/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs +++ b/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs @@ -24346,7 +24346,7 @@ pub fn vsqrtq_f64(a: float64x2_t) -> float64x2_t { #[cfg(not(target_arch = "arm64ec"))] #[cfg_attr(test, assert_instr(fsqrt))] pub fn vsqrth_f16(a: f16) -> f16 { - sqrtf16(a) + sqrt(a) } #[doc = "Shift Right and Insert (immediate)"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vsri_n_s8)"] diff --git a/library/stdarch/crates/core_arch/src/wasm32/mod.rs b/library/stdarch/crates/core_arch/src/wasm32/mod.rs index 48d05918208c7..fac7ac7b204b4 100644 --- a/library/stdarch/crates/core_arch/src/wasm32/mod.rs +++ b/library/stdarch/crates/core_arch/src/wasm32/mod.rs @@ -100,7 +100,7 @@ pub fn f32_nearest(a: f32) -> f32 { #[must_use = "method returns a new number and does not mutate the original value"] #[unstable(feature = "wasm_numeric_instr", issue = "133908")] pub fn f32_sqrt(a: f32) -> f32 { - crate::intrinsics::sqrtf32(a) + crate::intrinsics::sqrt(a) } /// Generates the [`f64.ceil`] instruction, returning the smallest integer greater than or equal to `a`. @@ -171,5 +171,5 @@ pub fn f64_nearest(a: f64) -> f64 { #[must_use = "method returns a new number and does not mutate the original value"] #[unstable(feature = "wasm_numeric_instr", issue = "133908")] pub fn f64_sqrt(a: f64) -> f64 { - crate::intrinsics::sqrtf64(a) + crate::intrinsics::sqrt(a) } diff --git a/library/stdarch/crates/core_arch/src/x86/sse.rs b/library/stdarch/crates/core_arch/src/x86/sse.rs index 67675fd696543..0322d9c655ed6 100644 --- a/library/stdarch/crates/core_arch/src/x86/sse.rs +++ b/library/stdarch/crates/core_arch/src/x86/sse.rs @@ -3,7 +3,7 @@ use crate::{ core_arch::{simd::*, x86::*}, intrinsics::simd::*, - intrinsics::sqrtf32, + intrinsics::sqrt, mem, ptr, }; @@ -123,7 +123,7 @@ pub const fn _mm_div_ps(a: __m128, b: __m128) -> __m128 { #[cfg_attr(test, assert_instr(sqrtss))] #[stable(feature = "simd_x86", since = "1.27.0")] pub fn _mm_sqrt_ss(a: __m128) -> __m128 { - unsafe { simd_insert!(a, 0, sqrtf32(_mm_cvtss_f32(a))) } + unsafe { simd_insert!(a, 0, sqrt(_mm_cvtss_f32(a))) } } /// Returns the square root of packed single-precision (32-bit) floating-point diff --git a/library/stdarch/crates/core_arch/src/x86/sse2.rs b/library/stdarch/crates/core_arch/src/x86/sse2.rs index 2c61b095c6b24..eef856fea6705 100644 --- a/library/stdarch/crates/core_arch/src/x86/sse2.rs +++ b/library/stdarch/crates/core_arch/src/x86/sse2.rs @@ -6,7 +6,7 @@ use stdarch_test::assert_instr; use crate::{ core_arch::{simd::*, x86::*}, intrinsics::simd::*, - intrinsics::sqrtf64, + intrinsics::sqrt, mem, ptr, }; @@ -1917,7 +1917,7 @@ pub const fn _mm_mul_pd(a: __m128d, b: __m128d) -> __m128d { #[cfg_attr(test, assert_instr(sqrtsd))] #[stable(feature = "simd_x86", since = "1.27.0")] pub fn _mm_sqrt_sd(a: __m128d, b: __m128d) -> __m128d { - unsafe { simd_insert!(a, 0, sqrtf64(_mm_cvtsd_f64(b))) } + unsafe { simd_insert!(a, 0, sqrt(_mm_cvtsd_f64(b))) } } /// Returns a new vector with the square root of each of the values in `a`. diff --git a/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml b/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml index 1558be0e373b0..af122e40265d4 100644 --- a/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml +++ b/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml @@ -8228,7 +8228,7 @@ intrinsics: types: - [f16, 'h_'] compose: - - FnCall: [sqrtf16, [a], []] + - FnCall: [sqrt, [a], []] - name: "vrsqrts{type[0]}" doc: "Floating-point reciprocal square root step" diff --git a/src/tools/clippy/clippy_utils/src/sym.rs b/src/tools/clippy/clippy_utils/src/sym.rs index 801a68d87dfca..db804f7d3dec1 100644 --- a/src/tools/clippy/clippy_utils/src/sym.rs +++ b/src/tools/clippy/clippy_utils/src/sym.rs @@ -563,7 +563,6 @@ generate! { split_whitespace, splitn, splitn_mut, - sqrt, starts_with, std_detect, step_by, diff --git a/src/tools/miri/src/intrinsics/math.rs b/src/tools/miri/src/intrinsics/math.rs index d60959ea73835..1ae0f603b7c01 100644 --- a/src/tools/miri/src/intrinsics/math.rs +++ b/src/tools/miri/src/intrinsics/math.rs @@ -100,10 +100,17 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { match intrinsic_name { // Operations we can do with soft-floats. - "sqrtf16" => sqrt::(this, args, dest)?, - "sqrtf32" => sqrt::(this, args, dest)?, - "sqrtf64" => sqrt::(this, args, dest)?, - "sqrtf128" => sqrt::(this, args, dest)?, + "sqrt" => { + let ty::Float(float_ty) = *generic_args.type_at(0).kind() else { + bug!("`sqrt` intrinsic called on non-float type"); + }; + match float_ty { + FloatTy::F16 => sqrt::(this, args, dest)?, + FloatTy::F32 => sqrt::(this, args, dest)?, + FloatTy::F64 => sqrt::(this, args, dest)?, + FloatTy::F128 => sqrt::(this, args, dest)?, + } + } #[rustfmt::skip] | "fadd_fast" diff --git a/tests/codegen-llvm/intrinsic-no-unnamed-attr.rs b/tests/codegen-llvm/intrinsic-no-unnamed-attr.rs index 255f20e6ff640..a2347cb01bb34 100644 --- a/tests/codegen-llvm/intrinsic-no-unnamed-attr.rs +++ b/tests/codegen-llvm/intrinsic-no-unnamed-attr.rs @@ -2,10 +2,10 @@ #![feature(core_intrinsics)] -use std::intrinsics::sqrtf32; +use std::intrinsics::sqrt; // CHECK: @llvm.sqrt.f32(float) #{{[0-9]*}} fn main() { - sqrtf32(0.0f32); + sqrt(0.0f32); } From 605acfbdcce334a121d63a478744797205fdf144 Mon Sep 17 00:00:00 2001 From: N1ark Date: Sun, 6 Sep 2026 20:28:41 +0100 Subject: [PATCH 4/8] Make powf generic --- .../src/intrinsics/mod.rs | 44 ++++++++++++-- .../rustc_codegen_gcc/src/intrinsic/mod.rs | 58 +++++++++--------- compiler/rustc_codegen_llvm/src/intrinsic.rs | 7 +-- compiler/rustc_codegen_llvm/src/lib.rs | 2 +- .../rustc_hir_analysis/src/check/intrinsic.rs | 12 +--- compiler/rustc_span/src/symbol.rs | 5 +- library/core/src/intrinsics/mod.rs | 59 ++++++------------- library/std/src/num/f128.rs | 2 +- library/std/src/num/f16.rs | 2 +- library/std/src/num/f32.rs | 2 +- library/std/src/num/f64.rs | 2 +- src/tools/clippy/clippy_utils/src/sym.rs | 1 - src/tools/miri/src/intrinsics/math.rs | 15 +++-- src/tools/miri/src/math.rs | 2 +- 14 files changed, 109 insertions(+), 104 deletions(-) diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index cd2b5de9d07e1..132d683855c53 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -327,11 +327,6 @@ fn codegen_float_intrinsic_call<'tcx>( sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64, types::F64), // compiler-builtins sym::powif128 => ("__powitf2", 2, fx.tcx.types.f128, types::F128), // compiler-builtins - sym::powf16 => return false, // has a fallback via f32 - sym::powf32 => ("powf", 2, fx.tcx.types.f32, types::F32), - sym::powf64 => ("pow", 2, fx.tcx.types.f64, types::F64), - sym::powf128 => ("powf128", 2, fx.tcx.types.f128, types::F128), - sym::fmaf16 => return false, // has a fallback via f64 sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32, types::F32), sym::fmaf64 => ("fma", 3, fx.tcx.types.f64, types::F64), @@ -1135,6 +1130,45 @@ fn codegen_regular_intrinsic_call<'tcx>( ret.write_cvalue(fx, old); } + sym::powf => { + intrinsic_args!(fx, args => (arg1, arg2); intrinsic); + let layout = arg1.layout(); + let ty::Float(float_ty) = layout.ty.kind() else { + span_bug!( + source_info.span, + "expected float type for {:?} intrinsic: {:?}", + intrinsic, + layout.ty + ); + }; + use FloatTy::*; + use IntrinsicFallback::*; + let x = arg1.load_scalar(fx); + let y = arg2.load_scalar(fx); + let res = match (intrinsic, float_ty) { + (sym::powf, F32) => Fallback("powf"), + (sym::powf, F64) => Fallback("pow"), + (sym::powf, F128) => Fallback("powf128"), + + (_, F16) => { + // We use the intrinsic fallback bodies for the rest + return Err(Instance::new_raw(instance.def_id(), instance.args)); + } + + _ => unreachable!(), + }; + let val = match res { + Codegen(val) => val, + Fallback(name) => { + let ty = fx.clif_type(layout.ty).unwrap(); + let arg = AbiParam::new(ty); + fx.lib_call(name, vec![arg, arg], vec![arg], &[x, y])[0] + } + }; + let val = CValue::by_val(val, layout); + ret.write_cvalue(fx, val); + } + sym::fabs | sym::sqrt | sym::floor diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index fae22c963283f..c98b1c01528f4 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -60,8 +60,6 @@ fn get_simple_intrinsic<'gcc, 'tcx>( let gcc_name = match name { sym::powif32 => "__builtin_powif", sym::powif64 => "__builtin_powi", - sym::powf32 => "powf", - sym::powf64 => "pow", sym::fmaf32 => "fmaf", sym::fmaf64 => "fma", // FIXME: calling `fma` from libc without FMA target feature uses expensive software emulation @@ -75,7 +73,6 @@ fn get_simple_intrinsic<'gcc, 'tcx>( sym::maximumf128 => return float_intrinsic(cx, cx.type_f128(), "fmaximumf128"), sym::copysignf32 => "copysignf", sym::copysignf64 => "copysign", - sym::powf128 => return float_intrinsic(cx, cx.type_f128(), "powf128"), sym::abort => "abort", _ => return None, }; @@ -88,31 +85,30 @@ fn get_simple_function_f128<'gcc, 'tcx>( name: Symbol, ) -> Function<'gcc> { let f128_type = cx.type_f128(); - let func_name = match name { - sym::ceil => "ceilf128", - sym::cos => "cosf128", - sym::fabs => "fabsf128", - sym::exp => "expf128", - sym::exp2 => "exp2f128", - sym::floor => "floorf128", - sym::log => "logf128", - sym::log2 => "log2f128", - sym::log10 => "log10f128", - sym::trunc => "truncf128", - sym::round => "roundf128", - sym::round_ties_even => "roundevenf128", - sym::sin => "sinf128", - sym::sqrt => "sqrtf128", - _ => span_bug!(span, "used get_simple_function_f128 for non-unary f128 intrinsic"), + let (func_name, args): (&str, &[gccjit::Type<'_>]) = match name { + sym::ceil => ("ceilf128", &[f128_type]), + sym::cos => ("cosf128", &[f128_type]), + sym::fabs => ("fabsf128", &[f128_type]), + sym::exp => ("expf128", &[f128_type]), + sym::exp2 => ("exp2f128", &[f128_type]), + sym::floor => ("floorf128", &[f128_type]), + sym::log => ("logf128", &[f128_type]), + sym::log2 => ("log2f128", &[f128_type]), + sym::log10 => ("log10f128", &[f128_type]), + sym::trunc => ("truncf128", &[f128_type]), + sym::round => ("roundf128", &[f128_type]), + sym::round_ties_even => ("roundevenf128", &[f128_type]), + sym::sin => ("sinf128", &[f128_type]), + sym::sqrt => ("sqrtf128", &[f128_type]), + sym::powf => ("powf128", &[f128_type, f128_type]), + _ => span_bug!(span, "used get_simple_function_f128 for unsupported f128 intrinsic"), }; - cx.context.new_function( - None, - FunctionType::Extern, - f128_type, - &[cx.context.new_parameter(None, f128_type, "a")], - func_name, - false, - ) + let args: Vec<_> = args + .iter() + .enumerate() + .map(|(index, typ)| cx.context.new_parameter(None, *typ, format!("param{}", index))) + .collect(); + cx.context.new_function(None, FunctionType::Extern, f128_type, &args, func_name, false) } fn f16_builtin<'gcc, 'tcx>( @@ -132,7 +128,7 @@ fn f16_builtin<'gcc, 'tcx>( sym::log => "logf", sym::log2 => "log2f", sym::log10 => "log10f", - sym::powf16 => "__builtin_powf", + sym::powf => "__builtin_powf", sym::round => "__builtin_roundf", sym::round_ties_even => "__builtin_rintf", sym::sin => "sinf", @@ -202,7 +198,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc &args.iter().map(|arg| arg.immediate()).collect::>(), ) } - sym::copysignf16 | sym::powf16 => f16_builtin(self, name, args), + sym::copysignf16 => f16_builtin(self, name, args), sym::copysignf128 if self.cx.supports_f128_type => { let f128_type = self.cx.type_f128(); let func = self.cx.context.new_function( @@ -379,6 +375,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc } sym::fabs | sym::sqrt + | sym::powf | sym::floor | sym::ceil | sym::trunc @@ -403,6 +400,9 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc (sym::sqrt, F32) => self.context.get_builtin_function("sqrtf"), (sym::sqrt, F64) => self.context.get_builtin_function("sqrt"), + (sym::powf, F32) => self.context.get_builtin_function("powf"), + (sym::powf, F64) => self.context.get_builtin_function("pow"), + (sym::floor, F32) => self.context.get_builtin_function("floorf"), (sym::floor, F64) => self.context.get_builtin_function("floor"), diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 71a4c58a59ccd..8a7de3adefb12 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -61,11 +61,6 @@ fn call_simple_intrinsic<'ll, 'tcx>( sym::powif64 => ("llvm.powi", &[bx.type_f64(), bx.type_i32()]), sym::powif128 => ("llvm.powi", &[bx.type_f128(), bx.type_i32()]), - sym::powf16 => ("llvm.pow", &[bx.type_f16()]), - sym::powf32 => ("llvm.pow", &[bx.type_f32()]), - sym::powf64 => ("llvm.pow", &[bx.type_f64()]), - sym::powf128 => ("llvm.pow", &[bx.type_f128()]), - sym::fmaf16 => ("llvm.fma", &[bx.type_f16()]), sym::fmaf32 => ("llvm.fma", &[bx.type_f32()]), sym::fmaf64 => ("llvm.fma", &[bx.type_f64()]), @@ -535,6 +530,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::fabs | sym::sqrt + | sym::powf | sym::floor | sym::ceil | sym::trunc @@ -560,6 +556,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { let llvm_name = match name { sym::fabs => "llvm.fabs", sym::sqrt => "llvm.sqrt", + sym::powf => "llvm.pow", sym::floor => "llvm.floor", sym::ceil => "llvm.ceil", sym::trunc => "llvm.trunc", diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 7477d949cfd8c..6626a509ca8fc 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -339,7 +339,7 @@ impl CodegenBackend for LlvmCodegenBackend { // Fallback via libm, but the LLVM intrinsic is used instead. sym::sin, sym::cos, - sym::powf16, sym::powf32, sym::powf64, + sym::powf, sym::sqrt, sym::exp, sym::exp2, diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index 5ff81024d0fcb..93469f4365f2a 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -149,10 +149,7 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi | sym::offload_get_num_devices | sym::offset_of | sym::overflow_checks - | sym::powf16 - | sym::powf32 - | sym::powf64 - | sym::powf128 + | sym::powf | sym::powif16 | sym::powif32 | sym::powif64 @@ -397,11 +394,6 @@ pub(crate) fn check_intrinsic_type( sym::powif64 => (0, 0, vec![tcx.types.f64, tcx.types.i32], tcx.types.f64), sym::powif128 => (0, 0, vec![tcx.types.f128, tcx.types.i32], tcx.types.f128), - sym::powf16 => (0, 0, vec![tcx.types.f16, tcx.types.f16], tcx.types.f16), - sym::powf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32), - sym::powf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64), - sym::powf128 => (0, 0, vec![tcx.types.f128, tcx.types.f128], tcx.types.f128), - sym::fmaf16 => (0, 0, vec![tcx.types.f16, tcx.types.f16, tcx.types.f16], tcx.types.f16), sym::fmaf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32, tcx.types.f32], tcx.types.f32), sym::fmaf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64, tcx.types.f64], tcx.types.f64), @@ -460,6 +452,8 @@ pub(crate) fn check_intrinsic_type( sym::copysignf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64), sym::copysignf128 => (0, 0, vec![tcx.types.f128, tcx.types.f128], tcx.types.f128), + sym::powf => (1, 0, vec![param(0), param(0)], param(0)), + sym::volatile_load | sym::unaligned_volatile_load => { (1, 0, vec![Ty::new_imm_ptr(tcx, param(0))], param(0)) } diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 45d26cf0e1c33..1bbc65a7b8f13 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1574,10 +1574,7 @@ symbols! { powerpc, powerpc64, powerpc_target_feature, - powf16, - powf32, - powf64, - powf128, + powf, powif16, powif32, powif64, diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index 96ad8feb2d74f..c4bd7599db87a 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -1141,48 +1141,25 @@ intrinsic_dispatch_on_type! { f128 => { libm::maybe_available::cosf128(x) } } -/// Raises an `f16` to an `f16` power. -/// -/// The stabilized version of this intrinsic is -/// [`f16::powf`](../../std/primitive.f16.html#method.powf) -#[inline] -#[rustc_intrinsic] -#[rustc_nounwind] -pub fn powf16(a: f16, x: f16) -> f16 { - powf32(a as f32, x as f32) as f16 -} -/// Raises an `f32` to an `f32` power. -/// -/// The stabilized version of this intrinsic is -/// [`f32::powf`](../../std/primitive.f32.html#method.powf) -#[inline] -#[rustc_intrinsic] -#[rustc_nounwind] -pub fn powf32(a: f32, x: f32) -> f32 { - cfg_select! { - all(target_env = "msvc", target_arch = "x86") => powf64(a as f64, x as f64) as f32, - _ => libm::likely_available::powf(a, x), +intrinsic_dispatch_on_type! { + /// Raises a floating-point value to a power of the same type. + /// + /// The stabilized versions of this intrinsic are available on the float primitives via the + /// `powf` method. For example, [`f32::powf`](../../std/primitive.f32.html#method.powf). + #[rustc_nounwind] + #[inline] + #[rustc_intrinsic] + pub fn powf(a: T, x: T) -> T; + + f16 => { powf(a as f32, x as f32) as f16 } + f32 => { + cfg_select! { + all(target_env = "msvc", target_arch = "x86") => powf(a as f64, x as f64) as f32, + _ => libm::likely_available::powf(a, x), + } } -} -/// Raises an `f64` to an `f64` power. -/// -/// The stabilized version of this intrinsic is -/// [`f64::powf`](../../std/primitive.f64.html#method.powf) -#[inline] -#[rustc_intrinsic] -#[rustc_nounwind] -pub fn powf64(a: f64, x: f64) -> f64 { - libm::likely_available::pow(a, x) -} -/// Raises an `f128` to an `f128` power. -/// -/// The stabilized version of this intrinsic is -/// [`f128::powf`](../../std/primitive.f128.html#method.powf) -#[inline] -#[rustc_intrinsic] -#[rustc_nounwind] -pub fn powf128(a: f128, x: f128) -> f128 { - libm::maybe_available::powf128(a, x) + f64 => { libm::likely_available::pow(a, x) } + f128 => { libm::maybe_available::powf128(a, x) } } intrinsic_dispatch_on_type! { diff --git a/library/std/src/num/f128.rs b/library/std/src/num/f128.rs index adc06358a5149..6224c70af3ab8 100644 --- a/library/std/src/num/f128.rs +++ b/library/std/src/num/f128.rs @@ -50,7 +50,7 @@ impl f128 { #[unstable(feature = "f128", issue = "116909")] #[must_use = "method returns a new number and does not mutate the original value"] pub fn powf(self, n: f128) -> f128 { - intrinsics::powf128(self, n) + intrinsics::powf(self, n) } /// Returns `e^(self)`, (the exponential function). diff --git a/library/std/src/num/f16.rs b/library/std/src/num/f16.rs index 3b5354d62f131..045b562397895 100644 --- a/library/std/src/num/f16.rs +++ b/library/std/src/num/f16.rs @@ -50,7 +50,7 @@ impl f16 { #[unstable(feature = "f16", issue = "116909")] #[must_use = "method returns a new number and does not mutate the original value"] pub fn powf(self, n: f16) -> f16 { - intrinsics::powf16(self, n) + intrinsics::powf(self, n) } /// Returns `e^(self)`, (the exponential function). diff --git a/library/std/src/num/f32.rs b/library/std/src/num/f32.rs index ac3f2e263195d..6ec81fad245c2 100644 --- a/library/std/src/num/f32.rs +++ b/library/std/src/num/f32.rs @@ -354,7 +354,7 @@ impl f32 { #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn powf(self, n: f32) -> f32 { - intrinsics::powf32(self, n) + intrinsics::powf(self, n) } /// Returns the square root of a number. diff --git a/library/std/src/num/f64.rs b/library/std/src/num/f64.rs index 9b3086b1ce12e..89793fbb91be9 100644 --- a/library/std/src/num/f64.rs +++ b/library/std/src/num/f64.rs @@ -354,7 +354,7 @@ impl f64 { #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn powf(self, n: f64) -> f64 { - intrinsics::powf64(self, n) + intrinsics::powf(self, n) } /// Returns the square root of a number. diff --git a/src/tools/clippy/clippy_utils/src/sym.rs b/src/tools/clippy/clippy_utils/src/sym.rs index db804f7d3dec1..1568dec0645d2 100644 --- a/src/tools/clippy/clippy_utils/src/sym.rs +++ b/src/tools/clippy/clippy_utils/src/sym.rs @@ -475,7 +475,6 @@ generate! { pop_if, position, pow, - powf, powi, print_macro, println_macro, diff --git a/src/tools/miri/src/intrinsics/math.rs b/src/tools/miri/src/intrinsics/math.rs index 1ae0f603b7c01..19c929d1cd85f 100644 --- a/src/tools/miri/src/intrinsics/math.rs +++ b/src/tools/miri/src/intrinsics/math.rs @@ -194,10 +194,17 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { }; } - "powf16" => pow_intrinsic::(this, args, dest)?, - "powf32" => pow_intrinsic::(this, args, dest)?, - "powf64" => pow_intrinsic::(this, args, dest)?, - "powf128" => todo!("f128"), // FIXME(f128) + "powf" => { + let ty::Float(float_ty) = *generic_args.type_at(0).kind() else { + bug!("`powf` intrinsic called on non-float type"); + }; + match float_ty { + FloatTy::F16 => pow_intrinsic::(this, args, dest)?, + FloatTy::F32 => pow_intrinsic::(this, args, dest)?, + FloatTy::F64 => pow_intrinsic::(this, args, dest)?, + FloatTy::F128 => todo!("f128"), // FIXME(f128) + } + } "powif16" => powi_intrinsic::(this, args, dest)?, "powif32" => powi_intrinsic::(this, args, dest)?, diff --git a/src/tools/miri/src/math.rs b/src/tools/miri/src/math.rs index bbee903252602..82a7877d1670a 100644 --- a/src/tools/miri/src/math.rs +++ b/src/tools/miri/src/math.rs @@ -235,7 +235,7 @@ where /// - tanhf, tanh, atanf, atan, atan2f, atan2 /// - exp, exp2 /// - log, log2, log10 -/// - powf32, powf64 +/// - powf /// - erff, erf, erfcf, erfc /// - hypotf, hypot /// From 66614fa2a426f73ff94a80e2a9c19a8a5741a973 Mon Sep 17 00:00:00 2001 From: N1ark Date: Sun, 6 Sep 2026 22:40:07 +0100 Subject: [PATCH 5/8] Make `fma` and `fmuladd` generic --- .../src/intrinsics/mod.rs | 57 ++++++--- .../rustc_codegen_gcc/src/intrinsic/mod.rs | 43 +++---- compiler/rustc_codegen_llvm/src/intrinsic.rs | 14 +-- compiler/rustc_codegen_llvm/src/lib.rs | 2 +- .../src/interpret/intrinsics.rs | 31 ++--- .../rustc_hir_analysis/src/check/intrinsic.rs | 24 +--- compiler/rustc_span/src/symbol.rs | 10 +- .../libm/src/math/support/float_traits.rs | 7 +- library/core/src/intrinsics/bounds.rs | 4 +- library/core/src/intrinsics/mod.rs | 109 ++++-------------- library/core/src/num/f128.rs | 2 +- library/core/src/num/f16.rs | 2 +- library/core/src/num/f32.rs | 2 +- library/core/src/num/f64.rs | 2 +- .../core_arch/src/aarch64/neon/generated.rs | 10 +- .../crates/core_arch/src/x86/avx512f.rs | 50 ++++---- .../crates/core_arch/src/x86/avx512fp16.rs | 34 +++--- .../stdarch/crates/core_arch/src/x86/fma.rs | 18 +-- .../spec/neon/aarch64.spec.yml | 4 +- src/tools/miri/tests/pass/float.rs | 8 +- .../intrinsics/fmuladd_nondeterministic.rs | 6 +- tests/ui/intrinsics/intrinsic-fmuladd.rs | 36 +++--- 22 files changed, 200 insertions(+), 275 deletions(-) diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index 132d683855c53..0ad1a30b36d3f 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -327,18 +327,6 @@ fn codegen_float_intrinsic_call<'tcx>( sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64, types::F64), // compiler-builtins sym::powif128 => ("__powitf2", 2, fx.tcx.types.f128, types::F128), // compiler-builtins - sym::fmaf16 => return false, // has a fallback via f64 - sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32, types::F32), - sym::fmaf64 => ("fma", 3, fx.tcx.types.f64, types::F64), - sym::fmaf128 => ("fmaf128", 3, fx.tcx.types.f128, types::F128), - - // FIXME: calling `fma` from libc without FMA target feature uses expensive sofware - // emulation, use cranelift intrinsic analogous to llvm.fmuladd.*. - sym::fmuladdf16 => return false, // has a fallback - sym::fmuladdf32 => ("fmaf", 3, fx.tcx.types.f32, types::F32), - sym::fmuladdf64 => ("fma", 3, fx.tcx.types.f64, types::F64), - sym::fmuladdf128 => return false, // has a fallback - sym::copysignf16 => return false, // has a fallback sym::copysignf32 => ("copysignf", 2, fx.tcx.types.f32, types::F32), sym::copysignf64 => ("copysign", 2, fx.tcx.types.f64, types::F64), @@ -379,9 +367,6 @@ fn codegen_float_intrinsic_call<'tcx>( // FIXME(bytecodealliance/wasmtime#8312): Use native Cranelift operations // for `f16` and `f128` once the lowerings have been implemented in Cranelift. let val = match intrinsic { - sym::fmaf32 | sym::fmaf64 | sym::fmuladdf32 | sym::fmuladdf64 => { - fx.bcx.ins().fma(args[0], args[1], args[2]) - } sym::copysignf32 | sym::copysignf64 => fx.bcx.ins().fcopysign(args[0], args[1]), // These intrinsics aren't supported natively by Cranelift. @@ -1130,6 +1115,48 @@ fn codegen_regular_intrinsic_call<'tcx>( ret.write_cvalue(fx, old); } + sym::fma | sym::fmuladd => { + intrinsic_args!(fx, args => (arg1, arg2, arg3); intrinsic); + let layout = arg1.layout(); + let ty::Float(float_ty) = layout.ty.kind() else { + span_bug!( + source_info.span, + "expected float type for {:?} intrinsic: {:?}", + intrinsic, + layout.ty + ); + }; + use FloatTy::*; + use IntrinsicFallback::*; + let x = arg1.load_scalar(fx); + let y = arg2.load_scalar(fx); + let z = arg3.load_scalar(fx); + let res = match (intrinsic, float_ty) { + // FIXME: calling `fma` from libc without FMA target feature uses expensive + // sofware emulation, use cranelift intrinsic analogous to llvm.fmuladd.*. + (sym::fma | sym::fmuladd, F32 | F64) => Codegen(fx.bcx.ins().fma(x, y, z)), + + (sym::fma, F128) => Fallback("fmaf128"), + + (sym::fmuladd, F128) | (_, F16) => { + // We use the intrinsic fallback bodies for the rest + return Err(Instance::new_raw(instance.def_id(), instance.args)); + } + + _ => unreachable!(), + }; + let val = match res { + Codegen(val) => val, + Fallback(name) => { + let ty = fx.clif_type(layout.ty).unwrap(); + let arg = AbiParam::new(ty); + fx.lib_call(name, vec![arg, arg, arg], vec![arg], &[x, y, z])[0] + } + }; + let val = CValue::by_val(val, layout); + ret.write_cvalue(fx, val); + } + sym::powf => { intrinsic_args!(fx, args => (arg1, arg2); intrinsic); let layout = arg1.layout(); diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index c98b1c01528f4..d2327f3675818 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -60,11 +60,6 @@ fn get_simple_intrinsic<'gcc, 'tcx>( let gcc_name = match name { sym::powif32 => "__builtin_powif", sym::powif64 => "__builtin_powi", - sym::fmaf32 => "fmaf", - sym::fmaf64 => "fma", - // FIXME: calling `fma` from libc without FMA target feature uses expensive software emulation - sym::fmuladdf32 => "fmaf", // FIXME: use gcc intrinsic analogous to llvm.fmuladd.f32 - sym::fmuladdf64 => "fma", // FIXME: use gcc intrinsic analogous to llvm.fmuladd.f64 sym::minimumf32 => return float_intrinsic(cx, cx.type_f32(), "fminimumf"), sym::minimumf64 => return float_intrinsic(cx, cx.type_f64(), "fminimum"), sym::minimumf128 => return float_intrinsic(cx, cx.type_f128(), "fminimumf128"), @@ -101,6 +96,7 @@ fn get_simple_function_f128<'gcc, 'tcx>( sym::sin => ("sinf128", &[f128_type]), sym::sqrt => ("sqrtf128", &[f128_type]), sym::powf => ("powf128", &[f128_type, f128_type]), + sym::fma => ("fmaf128", &[f128_type, f128_type, f128_type]), _ => span_bug!(span, "used get_simple_function_f128 for unsupported f128 intrinsic"), }; let args: Vec<_> = args @@ -218,26 +214,6 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc &args.iter().map(|arg| arg.immediate()).collect::>(), ) } - sym::fmaf128 => { - let f128_type = self.cx.type_f128(); - let func = self.cx.context.new_function( - None, - FunctionType::Extern, - f128_type, - &[ - self.cx.context.new_parameter(None, f128_type, "a"), - self.cx.context.new_parameter(None, f128_type, "b"), - self.cx.context.new_parameter(None, f128_type, "c"), - ], - "fmaf128", - false, - ); - self.cx.context.new_call( - self.location, - func, - &args.iter().map(|arg| arg.immediate()).collect::>(), - ) - } sym::powif16 => { let func = self.cx.context.get_builtin_function("__builtin_powif"); let arg0 = self.cx.context.new_cast(None, args[0].immediate(), self.cx.type_f32()); @@ -376,6 +352,8 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc sym::fabs | sym::sqrt | sym::powf + | sym::fma + | sym::fmuladd | sym::floor | sym::ceil | sym::trunc @@ -403,6 +381,21 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc (sym::powf, F32) => self.context.get_builtin_function("powf"), (sym::powf, F64) => self.context.get_builtin_function("pow"), + (sym::fma, F32) => self.context.get_builtin_function("fmaf"), + (sym::fma, F64) => self.context.get_builtin_function("fma"), + + // FIXME: calling `fma` from libc without FMA target feature uses expensive + // software emulation. + // FIXME: use gcc intrinsics analogous to llvm.fmuladd.f32/f64. + (sym::fmuladd, F32) => self.context.get_builtin_function("fmaf"), + (sym::fmuladd, F64) => self.context.get_builtin_function("fma"), + + // These have no `f16` builtin and no `fmuladdf128`; use the fallback bodies. + (sym::fma | sym::fmuladd, F16) | (sym::fmuladd, F128) => { + let fallback = Instance::new_raw(instance.def_id(), instance.args); + return IntrinsicResult::Fallback(fallback); + } + (sym::floor, F32) => self.context.get_builtin_function("floorf"), (sym::floor, F64) => self.context.get_builtin_function("floor"), diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 8a7de3adefb12..2d904ae7fb2b0 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -61,16 +61,6 @@ fn call_simple_intrinsic<'ll, 'tcx>( sym::powif64 => ("llvm.powi", &[bx.type_f64(), bx.type_i32()]), sym::powif128 => ("llvm.powi", &[bx.type_f128(), bx.type_i32()]), - sym::fmaf16 => ("llvm.fma", &[bx.type_f16()]), - sym::fmaf32 => ("llvm.fma", &[bx.type_f32()]), - sym::fmaf64 => ("llvm.fma", &[bx.type_f64()]), - sym::fmaf128 => ("llvm.fma", &[bx.type_f128()]), - - sym::fmuladdf16 => ("llvm.fmuladd", &[bx.type_f16()]), - sym::fmuladdf32 => ("llvm.fmuladd", &[bx.type_f32()]), - sym::fmuladdf64 => ("llvm.fmuladd", &[bx.type_f64()]), - sym::fmuladdf128 => ("llvm.fmuladd", &[bx.type_f128()]), - sym::minimumf16 => ("llvm.minimum", &[bx.type_f16()]), sym::minimumf32 => ("llvm.minimum", &[bx.type_f32()]), // FIXME: LLVM currently mis-compile those intrinsics, re-enable them @@ -531,6 +521,8 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::fabs | sym::sqrt | sym::powf + | sym::fma + | sym::fmuladd | sym::floor | sym::ceil | sym::trunc @@ -557,6 +549,8 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::fabs => "llvm.fabs", sym::sqrt => "llvm.sqrt", sym::powf => "llvm.pow", + sym::fma => "llvm.fma", + sym::fmuladd => "llvm.fmuladd", sym::floor => "llvm.floor", sym::ceil => "llvm.ceil", sym::trunc => "llvm.trunc", diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 6626a509ca8fc..bf11f6558d312 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -348,10 +348,10 @@ impl CodegenBackend for LlvmCodegenBackend { sym::log2, sym::floor, sym::ceil, sym::trunc, sym::round, sym::round_ties_even, + sym::fma, // Fallback via f32 or f64, but the LLVM intrinsic is used instead. sym::powif16, - sym::fmaf16, sym::copysignf16, sym::copysignf32, sym::copysignf64, sym::copysignf128, ]; diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index acf5369fd0156..d18918ad582a1 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -635,21 +635,22 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self.write_scalar(out_val, dest)?; } - sym::fmaf16 => self.float_muladd_intrinsic::(args, dest, MulAddType::Fused)?, - sym::fmaf32 => self.float_muladd_intrinsic::(args, dest, MulAddType::Fused)?, - sym::fmaf64 => self.float_muladd_intrinsic::(args, dest, MulAddType::Fused)?, - sym::fmaf128 => self.float_muladd_intrinsic::(args, dest, MulAddType::Fused)?, - sym::fmuladdf16 => { - self.float_muladd_intrinsic::(args, dest, MulAddType::Nondeterministic)? - } - sym::fmuladdf32 => { - self.float_muladd_intrinsic::(args, dest, MulAddType::Nondeterministic)? - } - sym::fmuladdf64 => { - self.float_muladd_intrinsic::(args, dest, MulAddType::Nondeterministic)? - } - sym::fmuladdf128 => { - self.float_muladd_intrinsic::(args, dest, MulAddType::Nondeterministic)? + sym::fma | sym::fmuladd => { + let mul_add = if intrinsic_name == sym::fma { + MulAddType::Fused + } else { + MulAddType::Nondeterministic + }; + let ty = args[0].layout.ty; + let ty::Float(float_ty) = ty.kind() else { + span_bug!(self.cur_span(), "non-float type for float intrinsic: {ty}"); + }; + match float_ty { + FloatTy::F16 => self.float_muladd_intrinsic::(args, dest, mul_add)?, + FloatTy::F32 => self.float_muladd_intrinsic::(args, dest, mul_add)?, + FloatTy::F64 => self.float_muladd_intrinsic::(args, dest, mul_add)?, + FloatTy::F128 => self.float_muladd_intrinsic::(args, dest, mul_add)?, + } } sym::va_copy => { diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index 93469f4365f2a..e69d8fd296ce9 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -107,15 +107,9 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi | sym::field_representing_type_name | sym::field_representing_type_offset | sym::floor - | sym::fmaf16 - | sym::fmaf32 - | sym::fmaf64 - | sym::fmaf128 + | sym::fma | sym::fmul_algebraic - | sym::fmuladdf16 - | sym::fmuladdf32 - | sym::fmuladdf64 - | sym::fmuladdf128 + | sym::fmuladd | sym::forget | sym::frem_algebraic | sym::fsub_algebraic @@ -394,19 +388,7 @@ pub(crate) fn check_intrinsic_type( sym::powif64 => (0, 0, vec![tcx.types.f64, tcx.types.i32], tcx.types.f64), sym::powif128 => (0, 0, vec![tcx.types.f128, tcx.types.i32], tcx.types.f128), - sym::fmaf16 => (0, 0, vec![tcx.types.f16, tcx.types.f16, tcx.types.f16], tcx.types.f16), - sym::fmaf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32, tcx.types.f32], tcx.types.f32), - sym::fmaf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64, tcx.types.f64], tcx.types.f64), - sym::fmaf128 => { - (0, 0, vec![tcx.types.f128, tcx.types.f128, tcx.types.f128], tcx.types.f128) - } - - sym::fmuladdf16 => (0, 0, vec![tcx.types.f16, tcx.types.f16, tcx.types.f16], tcx.types.f16), - sym::fmuladdf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32, tcx.types.f32], tcx.types.f32), - sym::fmuladdf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64, tcx.types.f64], tcx.types.f64), - sym::fmuladdf128 => { - (0, 0, vec![tcx.types.f128, tcx.types.f128, tcx.types.f128], tcx.types.f128) - } + sym::fma | sym::fmuladd => (1, 0, vec![param(0), param(0), param(0)], param(0)), sym::exp | sym::exp2 diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 1bbc65a7b8f13..2b733aadd9ad3 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -972,19 +972,13 @@ symbols! { final_associated_functions, float_to_int_unchecked, floor, + fma, fma4_target_feature, - fmaf16, - fmaf32, - fmaf64, - fmaf128, fmt, fmt_debug, fmul_algebraic, fmul_fast, - fmuladdf16, - fmuladdf32, - fmuladdf64, - fmuladdf128, + fmuladd, fn_align, fn_body, fn_delegation, diff --git a/library/compiler-builtins/libm/src/math/support/float_traits.rs b/library/compiler-builtins/libm/src/math/support/float_traits.rs index 1bded45ea930e..b02dcef193b59 100644 --- a/library/compiler-builtins/libm/src/math/support/float_traits.rs +++ b/library/compiler-builtins/libm/src/math/support/float_traits.rs @@ -251,7 +251,6 @@ macro_rules! float_impl { $from_bits:path, $to_bits:path, $fma_fn:ident, - $fma_intrinsic:ident ) => { impl Float for $ty { type Int = $ity; @@ -368,7 +367,7 @@ macro_rules! float_impl { cfg_select_nofmt! { // fma is not yet available in `core` intrinsics_enabled => { - core::intrinsics::$fma_intrinsic(self, y, z) + core::intrinsics::fma(self, y, z) } _ => { super::super::$fma_fn(self, y, z) @@ -393,7 +392,6 @@ float_impl!( f16::from_bits, f16::to_bits, fmaf16, - fmaf16 ); float_impl!( f32, @@ -404,7 +402,6 @@ float_impl!( f32_from_bits, f32_to_bits, fmaf, - fmaf32 ); float_impl!( f64, @@ -415,7 +412,6 @@ float_impl!( f64_from_bits, f64_to_bits, fma, - fmaf64 ); #[cfg(f128_enabled)] float_impl!( @@ -427,7 +423,6 @@ float_impl!( f128::from_bits, f128::to_bits, fmaf128, - fmaf128 ); /* FIXME(msrv): vendor some things that are not const stable at our MSRV */ diff --git a/library/core/src/intrinsics/bounds.rs b/library/core/src/intrinsics/bounds.rs index 085e131035c52..9fc599e045b8f 100644 --- a/library/core/src/intrinsics/bounds.rs +++ b/library/core/src/intrinsics/bounds.rs @@ -45,7 +45,9 @@ impl ChangePointee for *const T { /// # Safety /// Must actually *be* such a type. #[rustc_const_unstable(feature = "core_intrinsics", issue = "none")] -pub const unsafe trait FloatPrimitive: Sized + Copy { +pub const unsafe trait FloatPrimitive: + Sized + Copy + [const] core::ops::Mul + [const] core::ops::Add +{ type UInt: const core::ops::BitOr + const core::ops::BitAnd + const core::ops::Not; diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index c4bd7599db87a..47865c0dce476 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -1267,47 +1267,31 @@ intrinsic_dispatch_on_type! { f128 => { libm::maybe_available::log2f128(x) } } -/// Returns `a * b + c` without rounding the intermediate result for `f16` values. -/// -/// The stabilized version of this intrinsic is -/// [`f16::mul_add`](../../std/primitive.f16.html#method.mul_add) -#[rustc_intrinsic_const_stable_indirect] -#[inline] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn fmaf16(a: f16, b: f16, c: f16) -> f16 { +intrinsic_dispatch_on_type! { + /// Returns `a * b + c` without rounding the intermediate result. + /// + /// The stabilized versions of this intrinsic are available on the float primitives via the + /// `mul_add` method. For example, + /// [`f32::mul_add`](../../std/primitive.f32.html#method.mul_add). + #[rustc_nounwind] + #[inline] + #[rustc_intrinsic_const_stable_indirect] + #[rustc_intrinsic] + // The fallback bodies call into `libm`, which is not available at compile time. This is fine + // because const-eval implements this intrinsic itself and never runs the fallback body. + #[rustc_do_not_const_check] + pub const fn fma(a: T, b: T, c: T) -> T; + // NOTE: f32 does not have sufficient precision, so use f64 instead. // see also https://github.com/llvm/llvm-project/issues/128450#issuecomment-2727540179. - fmaf64(a as f64, b as f64, c as f64) as f16 + f16 => { fma(a as f64, b as f64, c as f64) as f16 } + f32 => { libm::fmaf(a, b, c) } + f64 => { libm::fma(a, b, c) } + f128 => { libm::fmaf128(a, b, c) } } -/// Returns `a * b + c` without rounding the intermediate result for `f32` values. -/// -/// The stabilized version of this intrinsic is -/// [`f32::mul_add`](../../std/primitive.f32.html#method.mul_add) -#[rustc_intrinsic_const_stable_indirect] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn fmaf32(a: f32, b: f32, c: f32) -> f32; -/// Returns `a * b + c` without rounding the intermediate result for `f64` values. -/// -/// The stabilized version of this intrinsic is -/// [`f64::mul_add`](../../std/primitive.f64.html#method.mul_add) -#[rustc_intrinsic_const_stable_indirect] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn fmaf64(a: f64, b: f64, c: f64) -> f64; -/// Returns `a * b + c` without rounding the intermediate result for `f128` values. -/// -/// The stabilized version of this intrinsic is -/// [`f128::mul_add`](../../std/primitive.f128.html#method.mul_add) -#[rustc_intrinsic_const_stable_indirect] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn fmaf128(a: f128, b: f128, c: f128) -> f128; -/// Returns `a * b + c` for `f16` values, non-deterministically executing -/// either a fused multiply-add or two operations with rounding of the -/// intermediate result. +/// Returns `a * b + c`, non-deterministically executing either a fused multiply-add or two +/// operations with rounding of the intermediate result. /// /// The operation is fused if the code generator determines that target /// instruction set has support for a fused operation, and that the fused @@ -1318,55 +1302,8 @@ pub const fn fmaf128(a: f128, b: f128, c: f128) -> f128; #[inline] #[rustc_intrinsic] #[rustc_nounwind] -pub const fn fmuladdf16(a: f16, b: f16, c: f16) -> f16 { - a * b + c -} -/// Returns `a * b + c` for `f32` values, non-deterministically executing -/// either a fused multiply-add or two operations with rounding of the -/// intermediate result. -/// -/// The operation is fused if the code generator determines that target -/// instruction set has support for a fused operation, and that the fused -/// operation is more efficient than the equivalent, separate pair of mul -/// and add instructions. It is unspecified whether or not a fused operation -/// is selected, and that may depend on optimization level and context, for -/// example. -#[inline] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn fmuladdf32(a: f32, b: f32, c: f32) -> f32 { - a * b + c -} -/// Returns `a * b + c` for `f64` values, non-deterministically executing -/// either a fused multiply-add or two operations with rounding of the -/// intermediate result. -/// -/// The operation is fused if the code generator determines that target -/// instruction set has support for a fused operation, and that the fused -/// operation is more efficient than the equivalent, separate pair of mul -/// and add instructions. It is unspecified whether or not a fused operation -/// is selected, and that may depend on optimization level and context, for -/// example. -#[inline] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn fmuladdf64(a: f64, b: f64, c: f64) -> f64 { - a * b + c -} -/// Returns `a * b + c` for `f128` values, non-deterministically executing -/// either a fused multiply-add or two operations with rounding of the -/// intermediate result. -/// -/// The operation is fused if the code generator determines that target -/// instruction set has support for a fused operation, and that the fused -/// operation is more efficient than the equivalent, separate pair of mul -/// and add instructions. It is unspecified whether or not a fused operation -/// is selected, and that may depend on optimization level and context, for -/// example. -#[inline] -#[rustc_intrinsic] -#[rustc_nounwind] -pub const fn fmuladdf128(a: f128, b: f128, c: f128) -> f128 { +#[rustc_const_unstable(feature = "core_intrinsics", issue = "none")] +pub const fn fmuladd(a: T, b: T, c: T) -> T { a * b + c } diff --git a/library/core/src/num/f128.rs b/library/core/src/num/f128.rs index 00f4e959064c8..8f52058e8100a 100644 --- a/library/core/src/num/f128.rs +++ b/library/core/src/num/f128.rs @@ -1994,7 +1994,7 @@ impl f128 { #[unstable(feature = "f128", issue = "116909")] #[must_use = "method returns a new number and does not mutate the original value"] pub const fn mul_add(self, a: f128, b: f128) -> f128 { - intrinsics::fmaf128(self, a, b) + intrinsics::fma(self, a, b) } /// Calculates Euclidean division, the matching method for `rem_euclid`. diff --git a/library/core/src/num/f16.rs b/library/core/src/num/f16.rs index e05ff63293af9..dfd3df4746663 100644 --- a/library/core/src/num/f16.rs +++ b/library/core/src/num/f16.rs @@ -1980,7 +1980,7 @@ impl f16 { #[doc(alias = "fmaf16", alias = "fusedMultiplyAdd")] #[must_use = "method returns a new number and does not mutate the original value"] pub const fn mul_add(self, a: f16, b: f16) -> f16 { - intrinsics::fmaf16(self, a, b) + intrinsics::fma(self, a, b) } /// Calculates Euclidean division, the matching method for `rem_euclid`. diff --git a/library/core/src/num/f32.rs b/library/core/src/num/f32.rs index d9718bb381365..bfd0e6c697a2d 100644 --- a/library/core/src/num/f32.rs +++ b/library/core/src/num/f32.rs @@ -2138,7 +2138,7 @@ pub mod math { #[must_use = "method returns a new number and does not mutate the original value"] #[unstable(feature = "core_float_math", issue = "137578")] pub const fn mul_add(x: f32, y: f32, z: f32) -> f32 { - intrinsics::fmaf32(x, y, z) + intrinsics::fma(x, y, z) } /// Experimental version of `div_euclid` in `core`. See [`f32::div_euclid`] for details. diff --git a/library/core/src/num/f64.rs b/library/core/src/num/f64.rs index ac80eeb7f9d92..51e7c7f4d923b 100644 --- a/library/core/src/num/f64.rs +++ b/library/core/src/num/f64.rs @@ -2116,7 +2116,7 @@ pub mod math { #[unstable(feature = "core_float_math", issue = "137578")] #[must_use = "method returns a new number and does not mutate the original value"] pub const fn mul_add(x: f64, a: f64, b: f64) -> f64 { - intrinsics::fmaf64(x, a, b) + intrinsics::fma(x, a, b) } /// Experimental version of `div_euclid` in `core`. See [`f64::div_euclid`] for details. diff --git a/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs b/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs index 0fdd332933c2c..54c5fa5a5b207 100644 --- a/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs +++ b/library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs @@ -8793,7 +8793,7 @@ pub fn vfma_n_f64(a: float64x1_t, b: float64x1_t, c: f64) -> float64x1_t { pub fn vfmad_lane_f64(a: f64, b: f64, c: float64x1_t) -> f64 { static_assert!(LANE == 0); let c: f64 = vget_lane_f64::(c); - fmaf64(b, c, a) + fma(b, c, a) } #[doc = "Floating-point fused multiply-add to accumulator"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmah_f16)"] @@ -8803,7 +8803,7 @@ pub fn vfmad_lane_f64(a: f64, b: f64, c: float64x1_t) -> f64 { #[unstable(feature = "stdarch_neon_f16", issue = "136306")] #[cfg(not(target_arch = "arm64ec"))] pub fn vfmah_f16(a: f16, b: f16, c: f16) -> f16 { - fmaf16(b, c, a) + fma(b, c, a) } #[doc = "Floating-point fused multiply-add to accumulator"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmah_lane_f16)"] @@ -8874,7 +8874,7 @@ pub fn vfmaq_n_f64(a: float64x2_t, b: float64x2_t, c: f64) -> float64x2_t { pub fn vfmas_lane_f32(a: f32, b: f32, c: float32x2_t) -> f32 { static_assert_uimm_bits!(LANE, 1); let c: f32 = vget_lane_f32::(c); - fmaf32(b, c, a) + fma(b, c, a) } #[doc = "Floating-point fused multiply-add to accumulator"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmas_laneq_f32)"] @@ -8886,7 +8886,7 @@ pub fn vfmas_lane_f32(a: f32, b: f32, c: float32x2_t) -> f32 { pub fn vfmas_laneq_f32(a: f32, b: f32, c: float32x4_t) -> f32 { static_assert_uimm_bits!(LANE, 2); let c: f32 = vgetq_lane_f32::(c); - fmaf32(b, c, a) + fma(b, c, a) } #[doc = "Floating-point fused multiply-add to accumulator"] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmad_laneq_f64)"] @@ -8898,7 +8898,7 @@ pub fn vfmas_laneq_f32(a: f32, b: f32, c: float32x4_t) -> f32 { pub fn vfmad_laneq_f64(a: f64, b: f64, c: float64x2_t) -> f64 { static_assert_uimm_bits!(LANE, 1); let c: f64 = vgetq_lane_f64::(c); - fmaf64(b, c, a) + fma(b, c, a) } #[doc = "Floating-point fused Multiply-Add Long to accumulator (vector)."] #[doc = "[Arm's documentation](https://developer.arm.com/architectures/instruction-sets/intrinsics/vfmlal_high_f16)"] diff --git a/library/stdarch/crates/core_arch/src/x86/avx512f.rs b/library/stdarch/crates/core_arch/src/x86/avx512f.rs index a9e498abf9b5a..eb2ecf545abfd 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512f.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512f.rs @@ -2,7 +2,7 @@ use crate::{ arch::asm, core_arch::{simd::*, x86::*}, intrinsics::simd::*, - intrinsics::{fmaf32, fmaf64}, + intrinsics::fma, mem, ptr, }; @@ -39352,7 +39352,7 @@ pub const fn _mm_mask_fmadd_ss(a: __m128, k: __mmask8, b: __m128, c: __m128) -> if (k & 0b00000001) != 0 { let extractb: f32 = simd_extract!(b, 0); let extractc: f32 = simd_extract!(c, 0); - fmadd = fmaf32(fmadd, extractb, extractc); + fmadd = fma(fmadd, extractb, extractc); } simd_insert!(a, 0, fmadd) } @@ -39373,7 +39373,7 @@ pub const fn _mm_maskz_fmadd_ss(k: __mmask8, a: __m128, b: __m128, c: __m128) -> let extracta: f32 = simd_extract!(a, 0); let extractb: f32 = simd_extract!(b, 0); let extractc: f32 = simd_extract!(c, 0); - fmadd = fmaf32(extracta, extractb, extractc); + fmadd = fma(extracta, extractb, extractc); } simd_insert!(a, 0, fmadd) } @@ -39393,7 +39393,7 @@ pub const fn _mm_mask3_fmadd_ss(a: __m128, b: __m128, c: __m128, k: __mmask8) -> if (k & 0b00000001) != 0 { let extracta: f32 = simd_extract!(a, 0); let extractb: f32 = simd_extract!(b, 0); - fmadd = fmaf32(extracta, extractb, fmadd); + fmadd = fma(extracta, extractb, fmadd); } simd_insert!(c, 0, fmadd) } @@ -39413,7 +39413,7 @@ pub const fn _mm_mask_fmadd_sd(a: __m128d, k: __mmask8, b: __m128d, c: __m128d) if (k & 0b00000001) != 0 { let extractb: f64 = simd_extract!(b, 0); let extractc: f64 = simd_extract!(c, 0); - fmadd = fmaf64(fmadd, extractb, extractc); + fmadd = fma(fmadd, extractb, extractc); } simd_insert!(a, 0, fmadd) } @@ -39434,7 +39434,7 @@ pub const fn _mm_maskz_fmadd_sd(k: __mmask8, a: __m128d, b: __m128d, c: __m128d) let extracta: f64 = simd_extract!(a, 0); let extractb: f64 = simd_extract!(b, 0); let extractc: f64 = simd_extract!(c, 0); - fmadd = fmaf64(extracta, extractb, extractc); + fmadd = fma(extracta, extractb, extractc); } simd_insert!(a, 0, fmadd) } @@ -39454,7 +39454,7 @@ pub const fn _mm_mask3_fmadd_sd(a: __m128d, b: __m128d, c: __m128d, k: __mmask8) if (k & 0b00000001) != 0 { let extracta: f64 = simd_extract!(a, 0); let extractb: f64 = simd_extract!(b, 0); - fmadd = fmaf64(extracta, extractb, fmadd); + fmadd = fma(extracta, extractb, fmadd); } simd_insert!(c, 0, fmadd) } @@ -39475,7 +39475,7 @@ pub const fn _mm_mask_fmsub_ss(a: __m128, k: __mmask8, b: __m128, c: __m128) -> let extractb: f32 = simd_extract!(b, 0); let extractc: f32 = simd_extract!(c, 0); let extractc = -extractc; - fmsub = fmaf32(fmsub, extractb, extractc); + fmsub = fma(fmsub, extractb, extractc); } simd_insert!(a, 0, fmsub) } @@ -39497,7 +39497,7 @@ pub const fn _mm_maskz_fmsub_ss(k: __mmask8, a: __m128, b: __m128, c: __m128) -> let extractb: f32 = simd_extract!(b, 0); let extractc: f32 = simd_extract!(c, 0); let extractc = -extractc; - fmsub = fmaf32(extracta, extractb, extractc); + fmsub = fma(extracta, extractb, extractc); } simd_insert!(a, 0, fmsub) } @@ -39518,7 +39518,7 @@ pub const fn _mm_mask3_fmsub_ss(a: __m128, b: __m128, c: __m128, k: __mmask8) -> let extracta: f32 = simd_extract!(a, 0); let extractb: f32 = simd_extract!(b, 0); let extractc = -fmsub; - fmsub = fmaf32(extracta, extractb, extractc); + fmsub = fma(extracta, extractb, extractc); } simd_insert!(c, 0, fmsub) } @@ -39539,7 +39539,7 @@ pub const fn _mm_mask_fmsub_sd(a: __m128d, k: __mmask8, b: __m128d, c: __m128d) let extractb: f64 = simd_extract!(b, 0); let extractc: f64 = simd_extract!(c, 0); let extractc = -extractc; - fmsub = fmaf64(fmsub, extractb, extractc); + fmsub = fma(fmsub, extractb, extractc); } simd_insert!(a, 0, fmsub) } @@ -39561,7 +39561,7 @@ pub const fn _mm_maskz_fmsub_sd(k: __mmask8, a: __m128d, b: __m128d, c: __m128d) let extractb: f64 = simd_extract!(b, 0); let extractc: f64 = simd_extract!(c, 0); let extractc = -extractc; - fmsub = fmaf64(extracta, extractb, extractc); + fmsub = fma(extracta, extractb, extractc); } simd_insert!(a, 0, fmsub) } @@ -39582,7 +39582,7 @@ pub const fn _mm_mask3_fmsub_sd(a: __m128d, b: __m128d, c: __m128d, k: __mmask8) let extracta: f64 = simd_extract!(a, 0); let extractb: f64 = simd_extract!(b, 0); let extractc = -fmsub; - fmsub = fmaf64(extracta, extractb, extractc); + fmsub = fma(extracta, extractb, extractc); } simd_insert!(c, 0, fmsub) } @@ -39603,7 +39603,7 @@ pub const fn _mm_mask_fnmadd_ss(a: __m128, k: __mmask8, b: __m128, c: __m128) -> let extracta = -fnmadd; let extractb: f32 = simd_extract!(b, 0); let extractc: f32 = simd_extract!(c, 0); - fnmadd = fmaf32(extracta, extractb, extractc); + fnmadd = fma(extracta, extractb, extractc); } simd_insert!(a, 0, fnmadd) } @@ -39625,7 +39625,7 @@ pub const fn _mm_maskz_fnmadd_ss(k: __mmask8, a: __m128, b: __m128, c: __m128) - let extracta = -extracta; let extractb: f32 = simd_extract!(b, 0); let extractc: f32 = simd_extract!(c, 0); - fnmadd = fmaf32(extracta, extractb, extractc); + fnmadd = fma(extracta, extractb, extractc); } simd_insert!(a, 0, fnmadd) } @@ -39646,7 +39646,7 @@ pub const fn _mm_mask3_fnmadd_ss(a: __m128, b: __m128, c: __m128, k: __mmask8) - let extracta: f32 = simd_extract!(a, 0); let extracta = -extracta; let extractb: f32 = simd_extract!(b, 0); - fnmadd = fmaf32(extracta, extractb, fnmadd); + fnmadd = fma(extracta, extractb, fnmadd); } simd_insert!(c, 0, fnmadd) } @@ -39667,7 +39667,7 @@ pub const fn _mm_mask_fnmadd_sd(a: __m128d, k: __mmask8, b: __m128d, c: __m128d) let extracta = -fnmadd; let extractb: f64 = simd_extract!(b, 0); let extractc: f64 = simd_extract!(c, 0); - fnmadd = fmaf64(extracta, extractb, extractc); + fnmadd = fma(extracta, extractb, extractc); } simd_insert!(a, 0, fnmadd) } @@ -39689,7 +39689,7 @@ pub const fn _mm_maskz_fnmadd_sd(k: __mmask8, a: __m128d, b: __m128d, c: __m128d let extracta = -extracta; let extractb: f64 = simd_extract!(b, 0); let extractc: f64 = simd_extract!(c, 0); - fnmadd = fmaf64(extracta, extractb, extractc); + fnmadd = fma(extracta, extractb, extractc); } simd_insert!(a, 0, fnmadd) } @@ -39710,7 +39710,7 @@ pub const fn _mm_mask3_fnmadd_sd(a: __m128d, b: __m128d, c: __m128d, k: __mmask8 let extracta: f64 = simd_extract!(a, 0); let extracta = -extracta; let extractb: f64 = simd_extract!(b, 0); - fnmadd = fmaf64(extracta, extractb, fnmadd); + fnmadd = fma(extracta, extractb, fnmadd); } simd_insert!(c, 0, fnmadd) } @@ -39732,7 +39732,7 @@ pub const fn _mm_mask_fnmsub_ss(a: __m128, k: __mmask8, b: __m128, c: __m128) -> let extractb: f32 = simd_extract!(b, 0); let extractc: f32 = simd_extract!(c, 0); let extractc = -extractc; - fnmsub = fmaf32(extracta, extractb, extractc); + fnmsub = fma(extracta, extractb, extractc); } simd_insert!(a, 0, fnmsub) } @@ -39755,7 +39755,7 @@ pub const fn _mm_maskz_fnmsub_ss(k: __mmask8, a: __m128, b: __m128, c: __m128) - let extractb: f32 = simd_extract!(b, 0); let extractc: f32 = simd_extract!(c, 0); let extractc = -extractc; - fnmsub = fmaf32(extracta, extractb, extractc); + fnmsub = fma(extracta, extractb, extractc); } simd_insert!(a, 0, fnmsub) } @@ -39777,7 +39777,7 @@ pub const fn _mm_mask3_fnmsub_ss(a: __m128, b: __m128, c: __m128, k: __mmask8) - let extracta = -extracta; let extractb: f32 = simd_extract!(b, 0); let extractc = -fnmsub; - fnmsub = fmaf32(extracta, extractb, extractc); + fnmsub = fma(extracta, extractb, extractc); } simd_insert!(c, 0, fnmsub) } @@ -39799,7 +39799,7 @@ pub const fn _mm_mask_fnmsub_sd(a: __m128d, k: __mmask8, b: __m128d, c: __m128d) let extractb: f64 = simd_extract!(b, 0); let extractc: f64 = simd_extract!(c, 0); let extractc = -extractc; - fnmsub = fmaf64(extracta, extractb, extractc); + fnmsub = fma(extracta, extractb, extractc); } simd_insert!(a, 0, fnmsub) } @@ -39822,7 +39822,7 @@ pub const fn _mm_maskz_fnmsub_sd(k: __mmask8, a: __m128d, b: __m128d, c: __m128d let extractb: f64 = simd_extract!(b, 0); let extractc: f64 = simd_extract!(c, 0); let extractc = -extractc; - fnmsub = fmaf64(extracta, extractb, extractc); + fnmsub = fma(extracta, extractb, extractc); } simd_insert!(a, 0, fnmsub) } @@ -39844,7 +39844,7 @@ pub const fn _mm_mask3_fnmsub_sd(a: __m128d, b: __m128d, c: __m128d, k: __mmask8 let extracta = -extracta; let extractb: f64 = simd_extract!(b, 0); let extractc = -fnmsub; - fnmsub = fmaf64(extracta, extractb, extractc); + fnmsub = fma(extracta, extractb, extractc); } simd_insert!(c, 0, fnmsub) } diff --git a/library/stdarch/crates/core_arch/src/x86/avx512fp16.rs b/library/stdarch/crates/core_arch/src/x86/avx512fp16.rs index 869f577a55905..f7e94ad248164 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512fp16.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512fp16.rs @@ -1,6 +1,6 @@ use crate::arch::asm; use crate::core_arch::{simd::*, x86::*}; -use crate::intrinsics::{fmaf16, simd::*}; +use crate::intrinsics::{fma, simd::*}; use crate::ptr; /// Set packed half-precision (16-bit) floating-point elements in dst with the supplied values. @@ -5537,7 +5537,7 @@ pub const fn _mm_fmadd_sh(a: __m128h, b: __m128h, c: __m128h) -> __m128h { let extracta: f16 = simd_extract!(a, 0); let extractb: f16 = simd_extract!(b, 0); let extractc: f16 = simd_extract!(c, 0); - let r = fmaf16(extracta, extractb, extractc); + let r = fma(extracta, extractb, extractc); simd_insert!(a, 0, r) } } @@ -5559,7 +5559,7 @@ pub const fn _mm_mask_fmadd_sh(a: __m128h, k: __mmask8, b: __m128h, c: __m128h) if k & 1 != 0 { let extractb: f16 = simd_extract!(b, 0); let extractc: f16 = simd_extract!(c, 0); - fmadd = fmaf16(fmadd, extractb, extractc); + fmadd = fma(fmadd, extractb, extractc); } simd_insert!(a, 0, fmadd) } @@ -5582,7 +5582,7 @@ pub const fn _mm_mask3_fmadd_sh(a: __m128h, b: __m128h, c: __m128h, k: __mmask8) if k & 1 != 0 { let extracta: f16 = simd_extract!(a, 0); let extractb: f16 = simd_extract!(b, 0); - fmadd = fmaf16(extracta, extractb, fmadd); + fmadd = fma(extracta, extractb, fmadd); } simd_insert!(c, 0, fmadd) } @@ -5606,7 +5606,7 @@ pub const fn _mm_maskz_fmadd_sh(k: __mmask8, a: __m128h, b: __m128h, c: __m128h) let extracta: f16 = simd_extract!(a, 0); let extractb: f16 = simd_extract!(b, 0); let extractc: f16 = simd_extract!(c, 0); - fmadd = fmaf16(extracta, extractb, extractc); + fmadd = fma(extracta, extractb, extractc); } simd_insert!(a, 0, fmadd) } @@ -6052,7 +6052,7 @@ pub const fn _mm_fmsub_sh(a: __m128h, b: __m128h, c: __m128h) -> __m128h { let extracta: f16 = simd_extract!(a, 0); let extractb: f16 = simd_extract!(b, 0); let extractc: f16 = simd_extract!(c, 0); - let r = fmaf16(extracta, extractb, -extractc); + let r = fma(extracta, extractb, -extractc); simd_insert!(a, 0, r) } } @@ -6074,7 +6074,7 @@ pub const fn _mm_mask_fmsub_sh(a: __m128h, k: __mmask8, b: __m128h, c: __m128h) if k & 1 != 0 { let extractb: f16 = simd_extract!(b, 0); let extractc: f16 = simd_extract!(c, 0); - fmsub = fmaf16(fmsub, extractb, -extractc); + fmsub = fma(fmsub, extractb, -extractc); } simd_insert!(a, 0, fmsub) } @@ -6097,7 +6097,7 @@ pub const fn _mm_mask3_fmsub_sh(a: __m128h, b: __m128h, c: __m128h, k: __mmask8) if k & 1 != 0 { let extracta: f16 = simd_extract!(a, 0); let extractb: f16 = simd_extract!(b, 0); - fmsub = fmaf16(extracta, extractb, -fmsub); + fmsub = fma(extracta, extractb, -fmsub); } simd_insert!(c, 0, fmsub) } @@ -6121,7 +6121,7 @@ pub const fn _mm_maskz_fmsub_sh(k: __mmask8, a: __m128h, b: __m128h, c: __m128h) let extracta: f16 = simd_extract!(a, 0); let extractb: f16 = simd_extract!(b, 0); let extractc: f16 = simd_extract!(c, 0); - fmsub = fmaf16(extracta, extractb, -extractc); + fmsub = fma(extracta, extractb, -extractc); } simd_insert!(a, 0, fmsub) } @@ -6558,7 +6558,7 @@ pub const fn _mm_fnmadd_sh(a: __m128h, b: __m128h, c: __m128h) -> __m128h { let extracta: f16 = simd_extract!(a, 0); let extractb: f16 = simd_extract!(b, 0); let extractc: f16 = simd_extract!(c, 0); - let r = fmaf16(-extracta, extractb, extractc); + let r = fma(-extracta, extractb, extractc); simd_insert!(a, 0, r) } } @@ -6580,7 +6580,7 @@ pub const fn _mm_mask_fnmadd_sh(a: __m128h, k: __mmask8, b: __m128h, c: __m128h) if k & 1 != 0 { let extractb: f16 = simd_extract!(b, 0); let extractc: f16 = simd_extract!(c, 0); - fnmadd = fmaf16(-fnmadd, extractb, extractc); + fnmadd = fma(-fnmadd, extractb, extractc); } simd_insert!(a, 0, fnmadd) } @@ -6603,7 +6603,7 @@ pub const fn _mm_mask3_fnmadd_sh(a: __m128h, b: __m128h, c: __m128h, k: __mmask8 if k & 1 != 0 { let extracta: f16 = simd_extract!(a, 0); let extractb: f16 = simd_extract!(b, 0); - fnmadd = fmaf16(-extracta, extractb, fnmadd); + fnmadd = fma(-extracta, extractb, fnmadd); } simd_insert!(c, 0, fnmadd) } @@ -6627,7 +6627,7 @@ pub const fn _mm_maskz_fnmadd_sh(k: __mmask8, a: __m128h, b: __m128h, c: __m128h let extracta: f16 = simd_extract!(a, 0); let extractb: f16 = simd_extract!(b, 0); let extractc: f16 = simd_extract!(c, 0); - fnmadd = fmaf16(-extracta, extractb, extractc); + fnmadd = fma(-extracta, extractb, extractc); } simd_insert!(a, 0, fnmadd) } @@ -7072,7 +7072,7 @@ pub const fn _mm_fnmsub_sh(a: __m128h, b: __m128h, c: __m128h) -> __m128h { let extracta: f16 = simd_extract!(a, 0); let extractb: f16 = simd_extract!(b, 0); let extractc: f16 = simd_extract!(c, 0); - let r = fmaf16(-extracta, extractb, -extractc); + let r = fma(-extracta, extractb, -extractc); simd_insert!(a, 0, r) } } @@ -7094,7 +7094,7 @@ pub const fn _mm_mask_fnmsub_sh(a: __m128h, k: __mmask8, b: __m128h, c: __m128h) if k & 1 != 0 { let extractb: f16 = simd_extract!(b, 0); let extractc: f16 = simd_extract!(c, 0); - fnmsub = fmaf16(-fnmsub, extractb, -extractc); + fnmsub = fma(-fnmsub, extractb, -extractc); } simd_insert!(a, 0, fnmsub) } @@ -7117,7 +7117,7 @@ pub const fn _mm_mask3_fnmsub_sh(a: __m128h, b: __m128h, c: __m128h, k: __mmask8 if k & 1 != 0 { let extracta: f16 = simd_extract!(a, 0); let extractb: f16 = simd_extract!(b, 0); - fnmsub = fmaf16(-extracta, extractb, -fnmsub); + fnmsub = fma(-extracta, extractb, -fnmsub); } simd_insert!(c, 0, fnmsub) } @@ -7141,7 +7141,7 @@ pub const fn _mm_maskz_fnmsub_sh(k: __mmask8, a: __m128h, b: __m128h, c: __m128h let extracta: f16 = simd_extract!(a, 0); let extractb: f16 = simd_extract!(b, 0); let extractc: f16 = simd_extract!(c, 0); - fnmsub = fmaf16(-extracta, extractb, -extractc); + fnmsub = fma(-extracta, extractb, -extractc); } simd_insert!(a, 0, fnmsub) } diff --git a/library/stdarch/crates/core_arch/src/x86/fma.rs b/library/stdarch/crates/core_arch/src/x86/fma.rs index b95bb331dfb0b..91df46a5281eb 100644 --- a/library/stdarch/crates/core_arch/src/x86/fma.rs +++ b/library/stdarch/crates/core_arch/src/x86/fma.rs @@ -20,7 +20,7 @@ use crate::core_arch::x86::*; use crate::intrinsics::simd::{simd_fma, simd_neg}; -use crate::intrinsics::{fmaf32, fmaf64}; +use crate::intrinsics::fma; #[cfg(test)] use stdarch_test::assert_instr; @@ -93,7 +93,7 @@ pub const fn _mm_fmadd_sd(a: __m128d, b: __m128d, c: __m128d) -> __m128d { simd_insert!( a, 0, - fmaf64(_mm_cvtsd_f64(a), _mm_cvtsd_f64(b), _mm_cvtsd_f64(c)) + fma(_mm_cvtsd_f64(a), _mm_cvtsd_f64(b), _mm_cvtsd_f64(c)) ) } } @@ -114,7 +114,7 @@ pub const fn _mm_fmadd_ss(a: __m128, b: __m128, c: __m128) -> __m128 { simd_insert!( a, 0, - fmaf32(_mm_cvtss_f32(a), _mm_cvtss_f32(b), _mm_cvtss_f32(c)) + fma(_mm_cvtss_f32(a), _mm_cvtss_f32(b), _mm_cvtss_f32(c)) ) } } @@ -259,7 +259,7 @@ pub const fn _mm_fmsub_sd(a: __m128d, b: __m128d, c: __m128d) -> __m128d { simd_insert!( a, 0, - fmaf64(_mm_cvtsd_f64(a), _mm_cvtsd_f64(b), -_mm_cvtsd_f64(c)) + fma(_mm_cvtsd_f64(a), _mm_cvtsd_f64(b), -_mm_cvtsd_f64(c)) ) } } @@ -280,7 +280,7 @@ pub const fn _mm_fmsub_ss(a: __m128, b: __m128, c: __m128) -> __m128 { simd_insert!( a, 0, - fmaf32(_mm_cvtss_f32(a), _mm_cvtss_f32(b), -_mm_cvtss_f32(c)) + fma(_mm_cvtss_f32(a), _mm_cvtss_f32(b), -_mm_cvtss_f32(c)) ) } } @@ -425,7 +425,7 @@ pub const fn _mm_fnmadd_sd(a: __m128d, b: __m128d, c: __m128d) -> __m128d { simd_insert!( a, 0, - fmaf64(_mm_cvtsd_f64(a), -_mm_cvtsd_f64(b), _mm_cvtsd_f64(c)) + fma(_mm_cvtsd_f64(a), -_mm_cvtsd_f64(b), _mm_cvtsd_f64(c)) ) } } @@ -446,7 +446,7 @@ pub const fn _mm_fnmadd_ss(a: __m128, b: __m128, c: __m128) -> __m128 { simd_insert!( a, 0, - fmaf32(_mm_cvtss_f32(a), -_mm_cvtss_f32(b), _mm_cvtss_f32(c)) + fma(_mm_cvtss_f32(a), -_mm_cvtss_f32(b), _mm_cvtss_f32(c)) ) } } @@ -524,7 +524,7 @@ pub const fn _mm_fnmsub_sd(a: __m128d, b: __m128d, c: __m128d) -> __m128d { simd_insert!( a, 0, - fmaf64(_mm_cvtsd_f64(a), -_mm_cvtsd_f64(b), -_mm_cvtsd_f64(c)) + fma(_mm_cvtsd_f64(a), -_mm_cvtsd_f64(b), -_mm_cvtsd_f64(c)) ) } } @@ -546,7 +546,7 @@ pub const fn _mm_fnmsub_ss(a: __m128, b: __m128, c: __m128) -> __m128 { simd_insert!( a, 0, - fmaf32(_mm_cvtss_f32(a), -_mm_cvtss_f32(b), -_mm_cvtss_f32(c)) + fma(_mm_cvtss_f32(a), -_mm_cvtss_f32(b), -_mm_cvtss_f32(c)) ) } } diff --git a/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml b/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml index af122e40265d4..0ce96e81f5e96 100644 --- a/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml +++ b/library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml @@ -10263,7 +10263,7 @@ intrinsics: compose: - FnCall: [static_assert!, ['LANE == 0']] - Let: [c, "{type[0]}", {FnCall: ['vget{neon_type[1].lane_nox}', [c], [LANE]]}] - - FnCall: [fmaf64, [b, c, a]] + - FnCall: [fma, [b, c, a]] - name: "vfma{type[1]}" @@ -10279,7 +10279,7 @@ intrinsics: types: - ["f16", "h_f16"] compose: - - FnCall: [fmaf16, [b, c, a], []] + - FnCall: [fma, [b, c, a], []] - name: "vfmah_lane{type[2]}" diff --git a/src/tools/miri/tests/pass/float.rs b/src/tools/miri/tests/pass/float.rs index 8eee645dce69c..019b775a450b1 100644 --- a/src/tools/miri/tests/pass/float.rs +++ b/src/tools/miri/tests/pass/float.rs @@ -1504,23 +1504,23 @@ fn test_algebraic() { } fn test_fmuladd() { - use std::intrinsics::{fmuladdf16, fmuladdf32, fmuladdf64}; + use std::intrinsics::fmuladd; // FIXME(f128): add when supported #[inline(never)] fn test_operations_f16(a: f16, b: f16, c: f16) { - assert_approx_eq!(fmuladdf16(a, b, c), a * b + c); + assert_approx_eq!(fmuladd(a, b, c), a * b + c); } #[inline(never)] fn test_operations_f32(a: f32, b: f32, c: f32) { - assert_approx_eq!(fmuladdf32(a, b, c), a * b + c); + assert_approx_eq!(fmuladd(a, b, c), a * b + c); } #[inline(never)] fn test_operations_f64(a: f64, b: f64, c: f64) { - assert_approx_eq!(fmuladdf64(a, b, c), a * b + c); + assert_approx_eq!(fmuladd(a, b, c), a * b + c); } test_operations_f16(0.1, 0.2, 0.3); diff --git a/src/tools/miri/tests/pass/intrinsics/fmuladd_nondeterministic.rs b/src/tools/miri/tests/pass/intrinsics/fmuladd_nondeterministic.rs index abc156d49cbb6..21c47e73e8c1b 100644 --- a/src/tools/miri/tests/pass/intrinsics/fmuladd_nondeterministic.rs +++ b/src/tools/miri/tests/pass/intrinsics/fmuladd_nondeterministic.rs @@ -1,6 +1,6 @@ #![feature(core_intrinsics, portable_simd)] use std::intrinsics::simd::simd_relaxed_fma; -use std::intrinsics::{fmuladdf32, fmuladdf64}; +use std::intrinsics::fmuladd; use std::simd::prelude::*; #[path = "../../utils/mod.rs"] @@ -14,7 +14,7 @@ fn main() { let c = std::hint::black_box(-a * b); // It is unspecified whether the following operation is fused or not. The // following evaluates to 0.0 if unfused, and nonzero (-1.66e-18) if fused. - let x = fmuladdf64(a, b, c); + let x = fmuladd(a, b, c); x == 0.0 }); @@ -24,7 +24,7 @@ fn main() { let c = std::hint::black_box(-a * b); // It is unspecified whether the following operation is fused or not. The // following evaluates to 0.0 if unfused, and nonzero (-8.1956386e-10) if fused. - let x = fmuladdf32(a, b, c); + let x = fmuladd(a, b, c); x == 0.0 }); diff --git a/tests/ui/intrinsics/intrinsic-fmuladd.rs b/tests/ui/intrinsics/intrinsic-fmuladd.rs index ab4285590cb95..6fc197a6ae902 100644 --- a/tests/ui/intrinsics/intrinsic-fmuladd.rs +++ b/tests/ui/intrinsics/intrinsic-fmuladd.rs @@ -15,28 +15,28 @@ fn main() { let nan: f32 = f32::NAN; let inf: f32 = f32::INFINITY; let neg_inf: f32 = f32::NEG_INFINITY; - assert_approx_eq!(fmuladdf32(1.23, 4.5, 0.67), 6.205); - assert_approx_eq!(fmuladdf32(-1.23, -4.5, -0.67), 4.865); - assert_approx_eq!(fmuladdf32(0.0, 8.9, 1.2), 1.2); - assert_approx_eq!(fmuladdf32(3.4, -0.0, 5.6), 5.6); - assert!(fmuladdf32(nan, 7.8, 9.0).is_nan()); - assert_eq!(fmuladdf32(inf, 7.8, 9.0), inf); - assert_eq!(fmuladdf32(neg_inf, 7.8, 9.0), neg_inf); - assert_eq!(fmuladdf32(8.9, inf, 3.2), inf); - assert_eq!(fmuladdf32(-3.2, 2.4, neg_inf), neg_inf); + assert_approx_eq!(fmuladd(1.23f32, 4.5, 0.67), 6.205); + assert_approx_eq!(fmuladd(-1.23f32, -4.5, -0.67), 4.865); + assert_approx_eq!(fmuladd(0.0f32, 8.9, 1.2), 1.2); + assert_approx_eq!(fmuladd(3.4f32, -0.0, 5.6), 5.6); + assert!(fmuladd(nan, 7.8, 9.0).is_nan()); + assert_eq!(fmuladd(inf, 7.8, 9.0), inf); + assert_eq!(fmuladd(neg_inf, 7.8, 9.0), neg_inf); + assert_eq!(fmuladd(8.9, inf, 3.2), inf); + assert_eq!(fmuladd(-3.2, 2.4, neg_inf), neg_inf); } { let nan: f64 = f64::NAN; let inf: f64 = f64::INFINITY; let neg_inf: f64 = f64::NEG_INFINITY; - assert_approx_eq!(fmuladdf64(1.23, 4.5, 0.67), 6.205); - assert_approx_eq!(fmuladdf64(-1.23, -4.5, -0.67), 4.865); - assert_approx_eq!(fmuladdf64(0.0, 8.9, 1.2), 1.2); - assert_approx_eq!(fmuladdf64(3.4, -0.0, 5.6), 5.6); - assert!(fmuladdf64(nan, 7.8, 9.0).is_nan()); - assert_eq!(fmuladdf64(inf, 7.8, 9.0), inf); - assert_eq!(fmuladdf64(neg_inf, 7.8, 9.0), neg_inf); - assert_eq!(fmuladdf64(8.9, inf, 3.2), inf); - assert_eq!(fmuladdf64(-3.2, 2.4, neg_inf), neg_inf); + assert_approx_eq!(fmuladd(1.23f64, 4.5, 0.67), 6.205); + assert_approx_eq!(fmuladd(-1.23f64, -4.5, -0.67), 4.865); + assert_approx_eq!(fmuladd(0.0f64, 8.9, 1.2), 1.2); + assert_approx_eq!(fmuladd(3.4f64, -0.0, 5.6), 5.6); + assert!(fmuladd(nan, 7.8, 9.0).is_nan()); + assert_eq!(fmuladd(inf, 7.8, 9.0), inf); + assert_eq!(fmuladd(neg_inf, 7.8, 9.0), neg_inf); + assert_eq!(fmuladd(8.9, inf, 3.2), inf); + assert_eq!(fmuladd(-3.2, 2.4, neg_inf), neg_inf); } } From 5f1bc1c9a6586c2a9044f99f7c901050a8525fe2 Mon Sep 17 00:00:00 2001 From: N1ark Date: Mon, 7 Sep 2026 01:16:31 +0100 Subject: [PATCH 6/8] Make `powi` generic --- .../src/intrinsics/mod.rs | 47 +++++++-------- .../rustc_codegen_gcc/src/intrinsic/mod.rs | 60 +++++++++---------- compiler/rustc_codegen_llvm/src/intrinsic.rs | 16 ++--- compiler/rustc_codegen_llvm/src/lib.rs | 3 - .../rustc_hir_analysis/src/check/intrinsic.rs | 10 +--- compiler/rustc_span/src/symbol.rs | 5 +- library/core/src/intrinsics/mod.rs | 32 ++-------- library/core/src/num/f128.rs | 2 +- library/core/src/num/f16.rs | 2 +- library/core/src/num/f32.rs | 2 +- library/core/src/num/f64.rs | 2 +- src/tools/clippy/clippy_utils/src/sym.rs | 1 - src/tools/miri/src/intrinsics/math.rs | 15 +++-- 13 files changed, 83 insertions(+), 114 deletions(-) diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index 0ad1a30b36d3f..07a8710f86947 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -322,11 +322,6 @@ fn codegen_float_intrinsic_call<'tcx>( ret: CPlace<'tcx>, ) -> bool { let (name, arg_count, ty, clif_ty) = match intrinsic { - sym::powif16 => ("__powisf2", 2, fx.tcx.types.f16, types::F16), // compiler-builtins - sym::powif32 => ("__powisf2", 2, fx.tcx.types.f32, types::F32), // compiler-builtins - sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64, types::F64), // compiler-builtins - sym::powif128 => ("__powitf2", 2, fx.tcx.types.f128, types::F128), // compiler-builtins - sym::copysignf16 => return false, // has a fallback sym::copysignf32 => ("copysignf", 2, fx.tcx.types.f32, types::F32), sym::copysignf64 => ("copysign", 2, fx.tcx.types.f64, types::F64), @@ -369,25 +364,6 @@ fn codegen_float_intrinsic_call<'tcx>( let val = match intrinsic { sym::copysignf32 | sym::copysignf64 => fx.bcx.ins().fcopysign(args[0], args[1]), - // These intrinsics aren't supported natively by Cranelift. - // Lower them to a libcall. - sym::powif16 | sym::powif32 | sym::powif64 | sym::powif128 => { - let temp; - let (clif_ty, args) = if intrinsic == sym::powif16 { - temp = [codegen_f16_f128::f16_to_f32(fx, args[0]), args[1]]; - (types::F32, temp.as_slice()) - } else { - (clif_ty, args) - }; - let input_tys: Vec<_> = - vec![AbiParam::new(clif_ty), lib_call_arg_param(fx.tcx, types::I32, true)]; - let ret_val = fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], args)[0]; - if intrinsic == sym::powif16 { - codegen_f16_f128::f32_to_f16(fx, ret_val) - } else { - ret_val - } - } _ => { let input_tys: Vec<_> = args.iter().map(|_| AbiParam::new(clif_ty)).collect(); fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], args)[0] @@ -1157,7 +1133,7 @@ fn codegen_regular_intrinsic_call<'tcx>( ret.write_cvalue(fx, val); } - sym::powf => { + sym::powf | sym::powi => { intrinsic_args!(fx, args => (arg1, arg2); intrinsic); let layout = arg1.layout(); let ty::Float(float_ty) = layout.ty.kind() else { @@ -1177,6 +1153,27 @@ fn codegen_regular_intrinsic_call<'tcx>( (sym::powf, F64) => Fallback("pow"), (sym::powf, F128) => Fallback("powf128"), + // Handle these manually because of the i32 argument + (sym::powi, F32 | F64 | F128) => { + let ty = fx.clif_type(layout.ty).unwrap(); + let arg = AbiParam::new(ty); + let i32 = AbiParam::new(types::I32); + let name = match float_ty { + F32 => "__powisf2", // compiler-builtins + F64 => "__powidf2", // compiler-builtins + F128 => "__powitf2", // compiler-builtins + _ => unreachable!(), + }; + Codegen(fx.lib_call(name, vec![arg, i32], vec![arg], &[x, y])[0]) + } + (sym::powi, F16) => { + let x = codegen_f16_f128::f16_to_f32(fx, x); + let f32 = AbiParam::new(types::F32); + let i32 = AbiParam::new(types::I32); + let val = fx.lib_call("__powisf2", vec![f32, i32], vec![f32], &[x, y])[0]; + Codegen(codegen_f16_f128::f32_to_f16(fx, val)) + } + (_, F16) => { // We use the intrinsic fallback bodies for the rest return Err(Instance::new_raw(instance.def_id(), instance.args)); diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index d2327f3675818..f817ea16c01ac 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -58,8 +58,6 @@ fn get_simple_intrinsic<'gcc, 'tcx>( name: Symbol, ) -> Option> { let gcc_name = match name { - sym::powif32 => "__builtin_powif", - sym::powif64 => "__builtin_powi", sym::minimumf32 => return float_intrinsic(cx, cx.type_f32(), "fminimumf"), sym::minimumf64 => return float_intrinsic(cx, cx.type_f64(), "fminimum"), sym::minimumf128 => return float_intrinsic(cx, cx.type_f128(), "fminimumf128"), @@ -214,32 +212,6 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc &args.iter().map(|arg| arg.immediate()).collect::>(), ) } - sym::powif16 => { - let func = self.cx.context.get_builtin_function("__builtin_powif"); - let arg0 = self.cx.context.new_cast(None, args[0].immediate(), self.cx.type_f32()); - let args = [arg0, args[1].immediate()]; - let result = self.cx.context.new_call(None, func, &args); - self.cx.context.new_cast(None, result, self.cx.type_f16()) - } - sym::powif128 => { - let f128_type = self.cx.type_f128(); - let func = self.cx.context.new_function( - None, - FunctionType::Extern, - f128_type, - &[ - self.cx.context.new_parameter(None, f128_type, "a"), - self.cx.context.new_parameter(None, self.int_type, "b"), - ], - "__powitf2", - false, - ); - self.cx.context.new_call( - self.location, - func, - &args.iter().map(|arg| arg.immediate()).collect::>(), - ) - } sym::is_val_statically_known => { let a = args[0].immediate(); let builtin = self.context.get_builtin_function("__builtin_constant_p"); @@ -365,7 +337,8 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc | sym::log10 | sym::log2 | sym::sin - | sym::cos => 'float_unop: { + | sym::cos + | sym::powi => 'float_op: { let ty = args[0].layout.ty; let ty::Float(float_ty) = *ty.kind() else { span_bug!(span, "expected float type for fabs intrinsic: {:?}", ty); @@ -396,6 +369,33 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc return IntrinsicResult::Fallback(fallback); } + (sym::powi, F32) => self.context.get_builtin_function("__builtin_powif"), + (sym::powi, F64) => self.context.get_builtin_function("__builtin_powi"), + // Provided by `compiler-builtins`. + (sym::powi, F128) => { + let f128_type = self.cx.type_f128(); + self.cx.context.new_function( + None, + FunctionType::Extern, + f128_type, + &[ + self.cx.context.new_parameter(None, f128_type, "a"), + self.cx.context.new_parameter(None, self.int_type, "b"), + ], + "__powitf2", + false, + ) + } + // `f16` can't go through `f16_builtin` due to the integer argument. + (sym::powi, F16) => { + let func = self.cx.context.get_builtin_function("__builtin_powif"); + let arg0 = + self.cx.context.new_cast(None, args[0].immediate(), self.cx.type_f32()); + let args = [arg0, args[1].immediate()]; + let result = self.cx.context.new_call(None, func, &args); + break 'float_op self.cx.context.new_cast(None, result, self.cx.type_f16()); + } + (sym::floor, F32) => self.context.get_builtin_function("floorf"), (sym::floor, F64) => self.context.get_builtin_function("floor"), @@ -435,7 +435,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc (_, F32 | F64) => unreachable!(), - (_, F16) => break 'float_unop f16_builtin(self, name, args), + (_, F16) => break 'float_op f16_builtin(self, name, args), (_, F128) => { if !self.cx.supports_f128_type { // Fall back to default body diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 2d904ae7fb2b0..f93492fe17e85 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -56,11 +56,6 @@ fn call_simple_intrinsic<'ll, 'tcx>( args: &[OperandRef<'tcx, &'ll Value>], ) -> Option<&'ll Value> { let (base_name, type_params): (&'static str, &[&'ll Type]) = match name { - sym::powif16 => ("llvm.powi", &[bx.type_f16(), bx.type_i32()]), - sym::powif32 => ("llvm.powi", &[bx.type_f32(), bx.type_i32()]), - sym::powif64 => ("llvm.powi", &[bx.type_f64(), bx.type_i32()]), - sym::powif128 => ("llvm.powi", &[bx.type_f128(), bx.type_i32()]), - sym::minimumf16 => ("llvm.minimum", &[bx.type_f16()]), sym::minimumf32 => ("llvm.minimum", &[bx.type_f32()]), // FIXME: LLVM currently mis-compile those intrinsics, re-enable them @@ -534,7 +529,8 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { | sym::log10 | sym::log2 | sym::sin - | sym::cos => { + | sym::cos + | sym::powi => { let ty = args[0].layout.ty; let ty::Float(f) = ty.kind() else { span_bug!( @@ -567,11 +563,17 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::log2 => "llvm.log2", sym::sin => "llvm.sin", sym::cos => "llvm.cos", + sym::powi => "llvm.powi", + _ => bug!(), }; + + let params: &[&'ll Type] = + if name == sym::powi { &[llty, self.type_i32()] } else { &[llty] }; + self.call_intrinsic( llvm_name, - &[llty], + params, &args.iter().map(|arg| arg.immediate()).collect::>(), ) } diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index bf11f6558d312..4651a965dc5d8 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -350,9 +350,6 @@ impl CodegenBackend for LlvmCodegenBackend { sym::round, sym::round_ties_even, sym::fma, - // Fallback via f32 or f64, but the LLVM intrinsic is used instead. - sym::powif16, - sym::copysignf16, sym::copysignf32, sym::copysignf64, sym::copysignf128, ]; diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index e69d8fd296ce9..b6d35ddf933ad 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -144,10 +144,7 @@ fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -> hi | sym::offset_of | sym::overflow_checks | sym::powf - | sym::powif16 - | sym::powif32 - | sym::powif64 - | sym::powif128 + | sym::powi | sym::prefetch_read_data | sym::prefetch_read_instruction | sym::prefetch_write_data @@ -383,10 +380,7 @@ pub(crate) fn check_intrinsic_type( tcx.types.unit, ), - sym::powif16 => (0, 0, vec![tcx.types.f16, tcx.types.i32], tcx.types.f16), - sym::powif32 => (0, 0, vec![tcx.types.f32, tcx.types.i32], tcx.types.f32), - sym::powif64 => (0, 0, vec![tcx.types.f64, tcx.types.i32], tcx.types.f64), - sym::powif128 => (0, 0, vec![tcx.types.f128, tcx.types.i32], tcx.types.f128), + sym::powi => (1, 0, vec![param(0), tcx.types.i32], param(0)), sym::fma | sym::fmuladd => (1, 0, vec![param(0), param(0), param(0)], param(0)), diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 2b733aadd9ad3..c580769d0f06a 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -1569,10 +1569,7 @@ symbols! { powerpc64, powerpc_target_feature, powf, - powif16, - powif32, - powif64, - powif128, + powi, pre_dash_lto: "pre-lto", precise_capturing, precise_capturing_in_traits, diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index 47865c0dce476..4a475579cfaeb 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -1067,37 +1067,13 @@ intrinsic_dispatch_on_type! { f128 => { libm::sqrtf128(x) } } -/// Raises an `f16` to an integer power. +/// Raises a floating-point value to an integer power. /// -/// The stabilized version of this intrinsic is -/// [`f16::powi`](../../std/primitive.f16.html#method.powi) -#[inline] -#[rustc_intrinsic] -#[rustc_nounwind] -pub fn powif16(a: f16, x: i32) -> f16 { - powif32(a as f32, x) as f16 -} -/// Raises an `f32` to an integer power. -/// -/// The stabilized version of this intrinsic is -/// [`f32::powi`](../../std/primitive.f32.html#method.powi) -#[rustc_intrinsic] -#[rustc_nounwind] -pub fn powif32(a: f32, x: i32) -> f32; -/// Raises an `f64` to an integer power. -/// -/// The stabilized version of this intrinsic is -/// [`f64::powi`](../../std/primitive.f64.html#method.powi) -#[rustc_intrinsic] -#[rustc_nounwind] -pub fn powif64(a: f64, x: i32) -> f64; -/// Raises an `f128` to an integer power. -/// -/// The stabilized version of this intrinsic is -/// [`f128::powi`](../../std/primitive.f128.html#method.powi) +/// The stabilized versions of this intrinsic are available on the float primitives via the +/// `powi` method. For example, [`f32::powi`](../../std/primitive.f32.html#method.powi). #[rustc_intrinsic] #[rustc_nounwind] -pub fn powif128(a: f128, x: i32) -> f128; +pub fn powi(a: T, x: i32) -> T; intrinsic_dispatch_on_type! { /// Returns the sine of a floating-point value. diff --git a/library/core/src/num/f128.rs b/library/core/src/num/f128.rs index 8f52058e8100a..fafc279c683c9 100644 --- a/library/core/src/num/f128.rs +++ b/library/core/src/num/f128.rs @@ -2113,7 +2113,7 @@ impl f128 { #[unstable(feature = "f128", issue = "116909")] #[must_use = "method returns a new number and does not mutate the original value"] pub fn powi(self, n: i32) -> f128 { - intrinsics::powif128(self, n) + intrinsics::powi(self, n) } /// Returns the square root of a number. diff --git a/library/core/src/num/f16.rs b/library/core/src/num/f16.rs index dfd3df4746663..cdadf60690855 100644 --- a/library/core/src/num/f16.rs +++ b/library/core/src/num/f16.rs @@ -2099,7 +2099,7 @@ impl f16 { #[unstable(feature = "f16", issue = "116909")] #[must_use = "method returns a new number and does not mutate the original value"] pub fn powi(self, n: i32) -> f16 { - intrinsics::powif16(self, n) + intrinsics::powi(self, n) } /// Returns the square root of a number. diff --git a/library/core/src/num/f32.rs b/library/core/src/num/f32.rs index bfd0e6c697a2d..6b49c8631acfd 100644 --- a/library/core/src/num/f32.rs +++ b/library/core/src/num/f32.rs @@ -2229,7 +2229,7 @@ pub mod math { #[must_use = "method returns a new number and does not mutate the original value"] #[unstable(feature = "core_float_math", issue = "137578")] pub fn powi(x: f32, n: i32) -> f32 { - intrinsics::powif32(x, n) + intrinsics::powi(x, n) } /// Experimental version of `sqrt` in `core`. See [`f32::sqrt`] for details. diff --git a/library/core/src/num/f64.rs b/library/core/src/num/f64.rs index 51e7c7f4d923b..267fbafd1432b 100644 --- a/library/core/src/num/f64.rs +++ b/library/core/src/num/f64.rs @@ -2207,7 +2207,7 @@ pub mod math { #[unstable(feature = "core_float_math", issue = "137578")] #[must_use = "method returns a new number and does not mutate the original value"] pub fn powi(x: f64, n: i32) -> f64 { - intrinsics::powif64(x, n) + intrinsics::powi(x, n) } /// Experimental version of `sqrt` in `core`. See [`f64::sqrt`] for details. diff --git a/src/tools/clippy/clippy_utils/src/sym.rs b/src/tools/clippy/clippy_utils/src/sym.rs index 1568dec0645d2..01d75e4c4fd12 100644 --- a/src/tools/clippy/clippy_utils/src/sym.rs +++ b/src/tools/clippy/clippy_utils/src/sym.rs @@ -475,7 +475,6 @@ generate! { pop_if, position, pow, - powi, print_macro, println_macro, process_abort, diff --git a/src/tools/miri/src/intrinsics/math.rs b/src/tools/miri/src/intrinsics/math.rs index 19c929d1cd85f..96c7e881d1b66 100644 --- a/src/tools/miri/src/intrinsics/math.rs +++ b/src/tools/miri/src/intrinsics/math.rs @@ -206,10 +206,17 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> { } } - "powif16" => powi_intrinsic::(this, args, dest)?, - "powif32" => powi_intrinsic::(this, args, dest)?, - "powif64" => powi_intrinsic::(this, args, dest)?, - "powif128" => todo!("f128"), // FIXME(f128) + "powi" => { + let ty::Float(float_ty) = *generic_args.type_at(0).kind() else { + bug!("`powf` intrinsic called on non-float type"); + }; + match float_ty { + FloatTy::F16 => powi_intrinsic::(this, args, dest)?, + FloatTy::F32 => powi_intrinsic::(this, args, dest)?, + FloatTy::F64 => powi_intrinsic::(this, args, dest)?, + FloatTy::F128 => todo!("f128"), // FIXME(f128) + } + } _ => return interp_ok(EmulateItemResult::NotSupported), } From 33503c26c9b84a7a4883c0569d9cf79c1dc3d300 Mon Sep 17 00:00:00 2001 From: N1ark Date: Mon, 7 Sep 2026 10:27:38 +0100 Subject: [PATCH 7/8] Add test for `rustc_do_not_const_check` intrinsics --- tests/ui/consts/const-float-intrinsics.rs | 29 +++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 tests/ui/consts/const-float-intrinsics.rs diff --git a/tests/ui/consts/const-float-intrinsics.rs b/tests/ui/consts/const-float-intrinsics.rs new file mode 100644 index 0000000000000..236993073ae25 --- /dev/null +++ b/tests/ui/consts/const-float-intrinsics.rs @@ -0,0 +1,29 @@ +//@ check-pass + +// Check that the float intrinsics carrying `#[rustc_do_not_const_check]` can actually be called in +// a const context, for every float width. Their fallback bodies call into `libm`, but const-eval +// overrides them. + +#![feature(core_intrinsics, f16, f128)] + +use std::intrinsics::{ceil, floor, fma, round, round_ties_even, trunc}; + +macro_rules! check { + ($ty:ident) => { + const _: () = { + assert!(floor(-2.5 as $ty) == -3.0); + assert!(ceil(-2.5 as $ty) == -2.0); + assert!(trunc(-2.5 as $ty) == -2.0); + assert!(round_ties_even(2.5 as $ty) == 2.0); + assert!(round(2.5 as $ty) == 3.0); + assert!(fma(3.0 as $ty, 4.0 as $ty, 5.0 as $ty) == 17.0); + }; + }; +} + +check!(f16); +check!(f32); +check!(f64); +check!(f128); + +fn main() {} From 8d81bf914d271a952f0f768c8df7283b228d2e03 Mon Sep 17 00:00:00 2001 From: N1ark Date: Mon, 7 Sep 2026 11:51:40 +0100 Subject: [PATCH 8/8] Add a fallback to `powi` --- .../src/intrinsics/mod.rs | 23 +----------------- .../rustc_codegen_gcc/src/intrinsic/mod.rs | 16 +++---------- compiler/rustc_codegen_llvm/src/lib.rs | 3 ++- library/core/src/intrinsics/mod.rs | 24 ++++++++++++------- library/core/src/num/imp/builtins.rs | 16 +++++++++++++ library/core/src/num/imp/mod.rs | 1 + 6 files changed, 39 insertions(+), 44 deletions(-) create mode 100644 library/core/src/num/imp/builtins.rs diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index 07a8710f86947..45bf4a1aa89f8 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -1133,7 +1133,7 @@ fn codegen_regular_intrinsic_call<'tcx>( ret.write_cvalue(fx, val); } - sym::powf | sym::powi => { + sym::powf => { intrinsic_args!(fx, args => (arg1, arg2); intrinsic); let layout = arg1.layout(); let ty::Float(float_ty) = layout.ty.kind() else { @@ -1153,27 +1153,6 @@ fn codegen_regular_intrinsic_call<'tcx>( (sym::powf, F64) => Fallback("pow"), (sym::powf, F128) => Fallback("powf128"), - // Handle these manually because of the i32 argument - (sym::powi, F32 | F64 | F128) => { - let ty = fx.clif_type(layout.ty).unwrap(); - let arg = AbiParam::new(ty); - let i32 = AbiParam::new(types::I32); - let name = match float_ty { - F32 => "__powisf2", // compiler-builtins - F64 => "__powidf2", // compiler-builtins - F128 => "__powitf2", // compiler-builtins - _ => unreachable!(), - }; - Codegen(fx.lib_call(name, vec![arg, i32], vec![arg], &[x, y])[0]) - } - (sym::powi, F16) => { - let x = codegen_f16_f128::f16_to_f32(fx, x); - let f32 = AbiParam::new(types::F32); - let i32 = AbiParam::new(types::I32); - let val = fx.lib_call("__powisf2", vec![f32, i32], vec![f32], &[x, y])[0]; - Codegen(codegen_f16_f128::f32_to_f16(fx, val)) - } - (_, F16) => { // We use the intrinsic fallback bodies for the rest return Err(Instance::new_raw(instance.def_id(), instance.args)); diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index f817ea16c01ac..e1f64a322b122 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -371,20 +371,10 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc (sym::powi, F32) => self.context.get_builtin_function("__builtin_powif"), (sym::powi, F64) => self.context.get_builtin_function("__builtin_powi"), - // Provided by `compiler-builtins`. + // No GCC builtin; the intrinsic's fallback uses `compiler-builtins`. (sym::powi, F128) => { - let f128_type = self.cx.type_f128(); - self.cx.context.new_function( - None, - FunctionType::Extern, - f128_type, - &[ - self.cx.context.new_parameter(None, f128_type, "a"), - self.cx.context.new_parameter(None, self.int_type, "b"), - ], - "__powitf2", - false, - ) + let fallback = Instance::new_raw(instance.def_id(), instance.args); + return IntrinsicResult::Fallback(fallback); } // `f16` can't go through `f16_builtin` due to the integer argument. (sym::powi, F16) => { diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index 4651a965dc5d8..966290bf504b8 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -336,10 +336,11 @@ impl CodegenBackend for LlvmCodegenBackend { sym::integer_max, sym::integer_min, - // Fallback via libm, but the LLVM intrinsic is used instead. + // Fallback via libm/compiler-builtins, but the LLVM intrinsic is used instead. sym::sin, sym::cos, sym::powf, + sym::powi, sym::sqrt, sym::exp, sym::exp2, diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index 4a475579cfaeb..44d627f442321 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -55,7 +55,7 @@ use crate::ffi::{VaArgSafe, VaList}; use crate::marker::{ConstParamTy, DiscriminantKind, PointeeSized, Tuple}; -use crate::num::imp::libm; +use crate::num::imp::{builtins, libm}; use crate::{mem, ptr}; mod bounds; @@ -1067,13 +1067,21 @@ intrinsic_dispatch_on_type! { f128 => { libm::sqrtf128(x) } } -/// Raises a floating-point value to an integer power. -/// -/// The stabilized versions of this intrinsic are available on the float primitives via the -/// `powi` method. For example, [`f32::powi`](../../std/primitive.f32.html#method.powi). -#[rustc_intrinsic] -#[rustc_nounwind] -pub fn powi(a: T, x: i32) -> T; +intrinsic_dispatch_on_type! { + /// Raises a floating-point value to an integer power. + /// + /// The stabilized versions of this intrinsic are available on the float primitives via the + /// `powi` method. For example, [`f32::powi`](../../std/primitive.f32.html#method.powi). + #[rustc_nounwind] + #[inline] + #[rustc_intrinsic] + pub fn powi(a: T, x: i32) -> T; + + f16 => { powi(a as f32, x) as f16 } + f32 => { builtins::__powisf2(a, x) } + f64 => { builtins::__powidf2(a, x) } + f128 => { builtins::__powitf2(a, x) } +} intrinsic_dispatch_on_type! { /// Returns the sine of a floating-point value. diff --git a/library/core/src/num/imp/builtins.rs b/library/core/src/num/imp/builtins.rs new file mode 100644 index 0000000000000..609c7ab9e9326 --- /dev/null +++ b/library/core/src/num/imp/builtins.rs @@ -0,0 +1,16 @@ +//! Bindings to functions provided by `compiler-builtins`. +//! +//! These are always available as `compiler-builtins` provides implementations, although other +//! implementations from libraries like libgcc/libgcc_s and compiler-rt may end up getting chosen by +//! the linker. + +// SAFETY: These symbols are defined by `compiler-builtins`, which is linked into every Rust +// program. +#[allow(dead_code)] // This list reflects what is available rather than what is consumed. +unsafe extern "C" { + pub(crate) safe fn __powisf2(a: f32, b: i32) -> f32; + pub(crate) safe fn __powidf2(a: f64, b: i32) -> f64; + // PowerPC uses `kf` rather than `tf` for `f128`. + #[cfg_attr(any(target_arch = "powerpc", target_arch = "powerpc64"), link_name = "__powikf2")] + pub(crate) safe fn __powitf2(a: f128, b: i32) -> f128; +} diff --git a/library/core/src/num/imp/mod.rs b/library/core/src/num/imp/mod.rs index 6fccfd1c238ed..0634201b29b6f 100644 --- a/library/core/src/num/imp/mod.rs +++ b/library/core/src/num/imp/mod.rs @@ -11,6 +11,7 @@ pub mod diy_float; pub mod flt2dec; pub mod fmt; +pub(crate) mod builtins; pub(crate) mod int_bits; pub(crate) mod int_log10; pub(crate) mod int_sqrt;