diff options
Diffstat (limited to 'compiler/rustc_codegen_cranelift/src/codegen_f16_f128.rs')
| -rw-r--r-- | compiler/rustc_codegen_cranelift/src/codegen_f16_f128.rs | 284 |
1 files changed, 284 insertions, 0 deletions
diff --git a/compiler/rustc_codegen_cranelift/src/codegen_f16_f128.rs b/compiler/rustc_codegen_cranelift/src/codegen_f16_f128.rs new file mode 100644 index 00000000000..1e202be1f18 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/codegen_f16_f128.rs @@ -0,0 +1,284 @@ +use crate::prelude::*; + +pub(crate) fn f16_to_f32(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { + let (value, arg_ty) = + if fx.tcx.sess.target.vendor == "apple" && fx.tcx.sess.target.arch == "x86_64" { + ( + fx.bcx.ins().bitcast(types::I16, MemFlags::new(), value), + lib_call_arg_param(fx.tcx, types::I16, false), + ) + } else { + (value, AbiParam::new(types::F16)) + }; + fx.lib_call("__extendhfsf2", vec![arg_ty], vec![AbiParam::new(types::F32)], &[value])[0] +} + +fn f16_to_f64(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { + let ret = f16_to_f32(fx, value); + fx.bcx.ins().fpromote(types::F64, ret) +} + +pub(crate) fn f32_to_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { + let ret_ty = if fx.tcx.sess.target.vendor == "apple" && fx.tcx.sess.target.arch == "x86_64" { + types::I16 + } else { + types::F16 + }; + let ret = fx.lib_call( + "__truncsfhf2", + vec![AbiParam::new(types::F32)], + vec![AbiParam::new(ret_ty)], + &[value], + )[0]; + if ret_ty == types::I16 { fx.bcx.ins().bitcast(types::F16, MemFlags::new(), ret) } else { ret } +} + +fn f64_to_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { + let ret_ty = if fx.tcx.sess.target.vendor == "apple" && fx.tcx.sess.target.arch == "x86_64" { + types::I16 + } else { + types::F16 + }; + let ret = fx.lib_call( + "__truncdfhf2", + vec![AbiParam::new(types::F64)], + vec![AbiParam::new(ret_ty)], + &[value], + )[0]; + if ret_ty == types::I16 { fx.bcx.ins().bitcast(types::F16, MemFlags::new(), ret) } else { ret } +} + +pub(crate) fn fcmp(fx: &mut FunctionCx<'_, '_, '_>, cc: FloatCC, lhs: Value, rhs: Value) -> Value { + let ty = fx.bcx.func.dfg.value_type(lhs); + match ty { + types::F32 | types::F64 => fx.bcx.ins().fcmp(cc, lhs, rhs), + types::F16 => { + let lhs = f16_to_f32(fx, lhs); + let rhs = f16_to_f32(fx, rhs); + fx.bcx.ins().fcmp(cc, lhs, rhs) + } + types::F128 => { + let (name, int_cc) = match cc { + FloatCC::Equal => ("__eqtf2", IntCC::Equal), + FloatCC::NotEqual => ("__netf2", IntCC::NotEqual), + FloatCC::LessThan => ("__lttf2", IntCC::SignedLessThan), + FloatCC::LessThanOrEqual => ("__letf2", IntCC::SignedLessThanOrEqual), + FloatCC::GreaterThan => ("__gttf2", IntCC::SignedGreaterThan), + FloatCC::GreaterThanOrEqual => ("__getf2", IntCC::SignedGreaterThanOrEqual), + _ => unreachable!("not currently used in rustc_codegen_cranelift: {cc:?}"), + }; + let res = fx.lib_call( + name, + vec![AbiParam::new(types::F128), AbiParam::new(types::F128)], + // FIXME(rust-lang/compiler-builtins#919): This should be `I64` on non-AArch64 + // architectures, but switching it before compiler-builtins is fixed causes test + // failures. + vec![AbiParam::new(types::I32)], + &[lhs, rhs], + )[0]; + let zero = fx.bcx.ins().iconst(types::I32, 0); + let res = fx.bcx.ins().icmp(int_cc, res, zero); + res + } + _ => unreachable!("{ty:?}"), + } +} + +pub(crate) fn codegen_f128_binop( + fx: &mut FunctionCx<'_, '_, '_>, + bin_op: BinOp, + lhs: Value, + rhs: Value, +) -> Value { + let name = match bin_op { + BinOp::Add => "__addtf3", + BinOp::Sub => "__subtf3", + BinOp::Mul => "__multf3", + BinOp::Div => "__divtf3", + _ => unreachable!("handled in `codegen_float_binop`"), + }; + fx.lib_call( + name, + vec![AbiParam::new(types::F128), AbiParam::new(types::F128)], + vec![AbiParam::new(types::F128)], + &[lhs, rhs], + )[0] +} + +pub(crate) fn neg_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { + let bits = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), value); + let bits = fx.bcx.ins().bxor_imm(bits, 0x8000); + fx.bcx.ins().bitcast(types::F16, MemFlags::new(), bits) +} + +pub(crate) fn neg_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { + let bits = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), value); + let (low, high) = fx.bcx.ins().isplit(bits); + let high = fx.bcx.ins().bxor_imm(high, 0x8000_0000_0000_0000_u64 as i64); + let bits = fx.bcx.ins().iconcat(low, high); + fx.bcx.ins().bitcast(types::F128, MemFlags::new(), bits) +} + +pub(crate) fn abs_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { + let bits = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), value); + let bits = fx.bcx.ins().band_imm(bits, 0x7fff); + fx.bcx.ins().bitcast(types::F16, MemFlags::new(), bits) +} + +pub(crate) fn abs_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value { + let bits = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), value); + let (low, high) = fx.bcx.ins().isplit(bits); + let high = fx.bcx.ins().band_imm(high, 0x7fff_ffff_ffff_ffff_u64 as i64); + let bits = fx.bcx.ins().iconcat(low, high); + fx.bcx.ins().bitcast(types::F128, MemFlags::new(), bits) +} + +pub(crate) fn copysign_f16(fx: &mut FunctionCx<'_, '_, '_>, lhs: Value, rhs: Value) -> Value { + let lhs = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), lhs); + let rhs = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), rhs); + let res = fx.bcx.ins().band_imm(lhs, 0x7fff); + let sign = fx.bcx.ins().band_imm(rhs, 0x8000); + let res = fx.bcx.ins().bor(res, sign); + fx.bcx.ins().bitcast(types::F16, MemFlags::new(), res) +} + +pub(crate) fn copysign_f128(fx: &mut FunctionCx<'_, '_, '_>, lhs: Value, rhs: Value) -> Value { + let lhs = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), lhs); + let rhs = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), rhs); + let (low, lhs_high) = fx.bcx.ins().isplit(lhs); + let (_, rhs_high) = fx.bcx.ins().isplit(rhs); + let high = fx.bcx.ins().band_imm(lhs_high, 0x7fff_ffff_ffff_ffff_u64 as i64); + let sign = fx.bcx.ins().band_imm(rhs_high, 0x8000_0000_0000_0000_u64 as i64); + let high = fx.bcx.ins().bor(high, sign); + let res = fx.bcx.ins().iconcat(low, high); + fx.bcx.ins().bitcast(types::F128, MemFlags::new(), res) +} + +pub(crate) fn codegen_cast( + fx: &mut FunctionCx<'_, '_, '_>, + from: Value, + from_signed: bool, + to_ty: Type, + to_signed: bool, +) -> Value { + let from_ty = fx.bcx.func.dfg.value_type(from); + if from_ty.is_float() && to_ty.is_float() { + let name = match (from_ty, to_ty) { + (types::F16, types::F32) => return f16_to_f32(fx, from), + (types::F16, types::F64) => return f16_to_f64(fx, from), + (types::F16, types::F128) => "__extendhftf2", + (types::F32, types::F128) => "__extendsftf2", + (types::F64, types::F128) => "__extenddftf2", + (types::F128, types::F64) => "__trunctfdf2", + (types::F128, types::F32) => "__trunctfsf2", + (types::F128, types::F16) => "__trunctfhf2", + (types::F64, types::F16) => return f64_to_f16(fx, from), + (types::F32, types::F16) => return f32_to_f16(fx, from), + _ => unreachable!("{from_ty:?} -> {to_ty:?}"), + }; + fx.lib_call(name, vec![AbiParam::new(from_ty)], vec![AbiParam::new(to_ty)], &[from])[0] + } else if from_ty.is_int() && to_ty == types::F16 { + let res = clif_int_or_float_cast(fx, from, from_signed, types::F32, false); + f32_to_f16(fx, res) + } else if from_ty == types::F16 && to_ty.is_int() { + let from = f16_to_f32(fx, from); + clif_int_or_float_cast(fx, from, false, to_ty, to_signed) + } else if from_ty.is_int() && to_ty == types::F128 { + let (from, from_ty) = if from_ty.bits() < 32 { + (clif_int_or_float_cast(fx, from, from_signed, types::I32, from_signed), types::I32) + } else { + (from, from_ty) + }; + let name = format!( + "__float{sign}{size}itf", + sign = if from_signed { "" } else { "un" }, + size = match from_ty { + types::I32 => 's', + types::I64 => 'd', + types::I128 => 't', + _ => unreachable!("{from_ty:?}"), + }, + ); + fx.lib_call( + &name, + vec![lib_call_arg_param(fx.tcx, from_ty, from_signed)], + vec![AbiParam::new(to_ty)], + &[from], + )[0] + } else if from_ty == types::F128 && to_ty.is_int() { + let ret_ty = if to_ty.bits() < 32 { types::I32 } else { to_ty }; + let name = format!( + "__fix{sign}tf{size}i", + sign = if from_signed { "" } else { "un" }, + size = match ret_ty { + types::I32 => 's', + types::I64 => 'd', + types::I128 => 't', + _ => unreachable!("{from_ty:?}"), + }, + ); + let ret = + fx.lib_call(&name, vec![AbiParam::new(from_ty)], vec![AbiParam::new(to_ty)], &[from]) + [0]; + let val = if ret_ty == to_ty { + ret + } else { + let (min, max) = match (to_ty, to_signed) { + (types::I8, false) => (0, i64::from(u8::MAX)), + (types::I16, false) => (0, i64::from(u16::MAX)), + (types::I8, true) => (i64::from(i8::MIN as u32), i64::from(i8::MAX as u32)), + (types::I16, true) => (i64::from(i16::MIN as u32), i64::from(i16::MAX as u32)), + _ => unreachable!("{to_ty:?}"), + }; + let min_val = fx.bcx.ins().iconst(types::I32, min); + let max_val = fx.bcx.ins().iconst(types::I32, max); + + let val = if to_signed { + let has_underflow = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, ret, min); + let has_overflow = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThan, ret, max); + let bottom_capped = fx.bcx.ins().select(has_underflow, min_val, ret); + fx.bcx.ins().select(has_overflow, max_val, bottom_capped) + } else { + let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, ret, max); + fx.bcx.ins().select(has_overflow, max_val, ret) + }; + fx.bcx.ins().ireduce(to_ty, val) + }; + + if let Some(false) = fx.tcx.sess.opts.unstable_opts.saturating_float_casts { + return val; + } + + let is_not_nan = fcmp(fx, FloatCC::Equal, from, from); + let zero = type_zero_value(&mut fx.bcx, to_ty); + fx.bcx.ins().select(is_not_nan, val, zero) + } else { + unreachable!("{from_ty:?} -> {to_ty:?}"); + } +} + +pub(crate) fn fma_f16(fx: &mut FunctionCx<'_, '_, '_>, x: Value, y: Value, z: Value) -> Value { + let x = f16_to_f64(fx, x); + let y = f16_to_f64(fx, y); + let z = f16_to_f64(fx, z); + let res = fx.bcx.ins().fma(x, y, z); + f64_to_f16(fx, res) +} + +pub(crate) fn fmin_f128(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value { + fx.lib_call( + "fminimumf128", + vec![AbiParam::new(types::F128), AbiParam::new(types::F128)], + vec![AbiParam::new(types::F128)], + &[a, b], + )[0] +} + +pub(crate) fn fmax_f128(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value { + fx.lib_call( + "fmaximumf128", + vec![AbiParam::new(types::F128), AbiParam::new(types::F128)], + vec![AbiParam::new(types::F128)], + &[a, b], + )[0] +} |