//! Codegen `extern "platform-intrinsic"` intrinsics. use super::*; use crate::prelude::*; pub(super) fn codegen_simd_intrinsic_call<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, instance: Instance<'tcx>, args: &[mir::Operand<'tcx>], ret: CPlace<'tcx>, span: Span, ) { let def_id = instance.def_id(); let substs = instance.substs; let intrinsic = fx.tcx.item_name(def_id); intrinsic_match! { fx, intrinsic, substs, args, _ => { fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic)); }; simd_cast, (c a) { validate_simd_type!(fx, intrinsic, span, a.layout().ty); simd_for_each_lane(fx, a, ret, |fx, lane_layout, ret_lane_layout, lane| { let ret_lane_ty = fx.clif_type(ret_lane_layout.ty).unwrap(); let from_signed = type_sign(lane_layout.ty); let to_signed = type_sign(ret_lane_layout.ty); let ret_lane = clif_int_or_float_cast(fx, lane, from_signed, ret_lane_ty, to_signed); CValue::by_val(ret_lane, ret_lane_layout) }); }; simd_eq, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); simd_cmp!(fx, Equal|Equal(x, y) -> ret); }; simd_ne, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); simd_cmp!(fx, NotEqual|NotEqual(x, y) -> ret); }; simd_lt, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); simd_cmp!(fx, UnsignedLessThan|SignedLessThan|LessThan(x, y) -> ret); }; simd_le, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); simd_cmp!(fx, UnsignedLessThanOrEqual|SignedLessThanOrEqual|LessThanOrEqual(x, y) -> ret); }; simd_gt, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); simd_cmp!(fx, UnsignedGreaterThan|SignedGreaterThan|GreaterThan(x, y) -> ret); }; simd_ge, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); simd_cmp!( fx, UnsignedGreaterThanOrEqual|SignedGreaterThanOrEqual|GreaterThanOrEqual (x, y) -> ret ); }; // simd_shuffle32(x: T, y: T, idx: [u32; 32]) -> U _ if intrinsic.as_str().starts_with("simd_shuffle"), (c x, c y, o idx) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); let n: u16 = intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap(); assert_eq!(x.layout(), y.layout()); let layout = x.layout(); let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); assert_eq!(lane_ty, ret_lane_ty); assert_eq!(u64::from(n), ret_lane_count); let total_len = lane_count * 2; let indexes = { use rustc_middle::mir::interpret::*; let idx_const = crate::constant::mir_operand_get_const_val(fx, idx).expect("simd_shuffle* idx not const"); let idx_bytes = match idx_const { ConstValue::ByRef { alloc, offset } => { let size = Size::from_bytes(4 * ret_lane_count /* size_of([u32; ret_lane_count]) */); alloc.get_bytes(fx, alloc_range(offset, size)).unwrap() } _ => unreachable!("{:?}", idx_const), }; (0..ret_lane_count).map(|i| { let i = usize::try_from(i).unwrap(); let idx = rustc_middle::mir::interpret::read_target_uint( fx.tcx.data_layout.endian, &idx_bytes[4*i.. 4*i + 4], ).expect("read_target_uint"); u16::try_from(idx).expect("try_from u32") }).collect::>() }; for &idx in &indexes { assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len); } for (out_idx, in_idx) in indexes.into_iter().enumerate() { let in_lane = if u64::from(in_idx) < lane_count { x.value_lane(fx, in_idx.into()) } else { y.value_lane(fx, u64::from(in_idx) - lane_count) }; let out_lane = ret.place_lane(fx, u64::try_from(out_idx).unwrap()); out_lane.write_cvalue(fx, in_lane); } }; simd_insert, (c base, o idx, c val) { // FIXME validate let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) { idx_const } else { fx.tcx.sess.span_fatal( span, "Index argument for `simd_insert` is not a constant", ); }; let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const)); let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx); if idx >= lane_count.into() { fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count)); } ret.write_cvalue(fx, base); let ret_lane = ret.place_field(fx, mir::Field::new(idx.try_into().unwrap())); ret_lane.write_cvalue(fx, val); }; simd_extract, (c v, o idx) { validate_simd_type!(fx, intrinsic, span, v.layout().ty); let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) { idx_const } else { fx.tcx.sess.span_warn( span, "Index argument for `simd_extract` is not a constant", ); let res = crate::trap::trap_unimplemented_ret_value( fx, ret.layout(), "Index argument for `simd_extract` is not a constant", ); ret.write_cvalue(fx, res); return; }; let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const)); let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx); if idx >= lane_count.into() { fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count)); } let ret_lane = v.value_lane(fx, idx.try_into().unwrap()); ret.write_cvalue(fx, ret_lane); }; simd_neg, (c a) { validate_simd_type!(fx, intrinsic, span, a.layout().ty); simd_for_each_lane(fx, a, ret, |fx, lane_layout, ret_lane_layout, lane| { let ret_lane = match lane_layout.ty.kind() { ty::Int(_) => fx.bcx.ins().ineg(lane), ty::Float(_) => fx.bcx.ins().fneg(lane), _ => unreachable!(), }; CValue::by_val(ret_lane, ret_lane_layout) }); }; simd_fabs, (c a) { validate_simd_type!(fx, intrinsic, span, a.layout().ty); simd_for_each_lane(fx, a, ret, |fx, _lane_layout, ret_lane_layout, lane| { let ret_lane = fx.bcx.ins().fabs(lane); CValue::by_val(ret_lane, ret_lane_layout) }); }; simd_fsqrt, (c a) { validate_simd_type!(fx, intrinsic, span, a.layout().ty); simd_for_each_lane(fx, a, ret, |fx, _lane_layout, ret_lane_layout, lane| { let ret_lane = fx.bcx.ins().sqrt(lane); CValue::by_val(ret_lane, ret_lane_layout) }); }; simd_add, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); simd_int_flt_binop!(fx, iadd|fadd(x, y) -> ret); }; simd_sub, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); simd_int_flt_binop!(fx, isub|fsub(x, y) -> ret); }; simd_mul, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); simd_int_flt_binop!(fx, imul|fmul(x, y) -> ret); }; simd_div, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); simd_int_flt_binop!(fx, udiv|sdiv|fdiv(x, y) -> ret); }; simd_rem, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); simd_pair_for_each_lane(fx, x, y, ret, |fx, lane_layout, ret_lane_layout, x_lane, y_lane| { let res_lane = match lane_layout.ty.kind() { ty::Uint(_) => fx.bcx.ins().urem(x_lane, y_lane), ty::Int(_) => fx.bcx.ins().srem(x_lane, y_lane), ty::Float(FloatTy::F32) => fx.lib_call( "fmodf", vec![AbiParam::new(types::F32), AbiParam::new(types::F32)], vec![AbiParam::new(types::F32)], &[x_lane, y_lane], )[0], ty::Float(FloatTy::F64) => fx.lib_call( "fmod", vec![AbiParam::new(types::F64), AbiParam::new(types::F64)], vec![AbiParam::new(types::F64)], &[x_lane, y_lane], )[0], _ => unreachable!("{:?}", lane_layout.ty), }; CValue::by_val(res_lane, ret_lane_layout) }); }; simd_shl, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); simd_int_binop!(fx, ishl(x, y) -> ret); }; simd_shr, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); simd_int_binop!(fx, ushr|sshr(x, y) -> ret); }; simd_and, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); simd_int_binop!(fx, band(x, y) -> ret); }; simd_or, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); simd_int_binop!(fx, bor(x, y) -> ret); }; simd_xor, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); simd_int_binop!(fx, bxor(x, y) -> ret); }; simd_fma, (c a, c b, c c) { validate_simd_type!(fx, intrinsic, span, a.layout().ty); assert_eq!(a.layout(), b.layout()); assert_eq!(a.layout(), c.layout()); let layout = a.layout(); let (lane_count, _lane_ty) = layout.ty.simd_size_and_type(fx.tcx); let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); assert_eq!(lane_count, ret_lane_count); let ret_lane_layout = fx.layout_of(ret_lane_ty); for lane in 0..lane_count { let a_lane = a.value_lane(fx, lane).load_scalar(fx); let b_lane = b.value_lane(fx, lane).load_scalar(fx); let c_lane = c.value_lane(fx, lane).load_scalar(fx); let mul_lane = fx.bcx.ins().fmul(a_lane, b_lane); let res_lane = CValue::by_val(fx.bcx.ins().fadd(mul_lane, c_lane), ret_lane_layout); ret.place_lane(fx, lane).write_cvalue(fx, res_lane); } }; simd_fmin, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); simd_flt_binop!(fx, fmin(x, y) -> ret); }; simd_fmax, (c x, c y) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); simd_flt_binop!(fx, fmax(x, y) -> ret); }; simd_round, (c a) { validate_simd_type!(fx, intrinsic, span, a.layout().ty); simd_for_each_lane(fx, a, ret, |fx, lane_layout, ret_lane_layout, lane| { let res_lane = match lane_layout.ty.kind() { ty::Float(FloatTy::F32) => fx.lib_call( "roundf", vec![AbiParam::new(types::F32)], vec![AbiParam::new(types::F32)], &[lane], )[0], ty::Float(FloatTy::F64) => fx.lib_call( "round", vec![AbiParam::new(types::F64)], vec![AbiParam::new(types::F64)], &[lane], )[0], _ => unreachable!("{:?}", lane_layout.ty), }; CValue::by_val(res_lane, ret_lane_layout) }); }; simd_ceil, (c a) { validate_simd_type!(fx, intrinsic, span, a.layout().ty); simd_for_each_lane(fx, a, ret, |fx, _lane_layout, ret_lane_layout, lane| { let ret_lane = fx.bcx.ins().ceil(lane); CValue::by_val(ret_lane, ret_lane_layout) }); }; simd_floor, (c a) { validate_simd_type!(fx, intrinsic, span, a.layout().ty); simd_for_each_lane(fx, a, ret, |fx, _lane_layout, ret_lane_layout, lane| { let ret_lane = fx.bcx.ins().floor(lane); CValue::by_val(ret_lane, ret_lane_layout) }); }; simd_trunc, (c a) { validate_simd_type!(fx, intrinsic, span, a.layout().ty); simd_for_each_lane(fx, a, ret, |fx, _lane_layout, ret_lane_layout, lane| { let ret_lane = fx.bcx.ins().trunc(lane); CValue::by_val(ret_lane, ret_lane_layout) }); }; simd_reduce_add_ordered | simd_reduce_add_unordered, (c v, v acc) { validate_simd_type!(fx, intrinsic, span, v.layout().ty); simd_reduce(fx, v, Some(acc), ret, |fx, lane_layout, a, b| { if lane_layout.ty.is_floating_point() { fx.bcx.ins().fadd(a, b) } else { fx.bcx.ins().iadd(a, b) } }); }; simd_reduce_mul_ordered | simd_reduce_mul_unordered, (c v, v acc) { validate_simd_type!(fx, intrinsic, span, v.layout().ty); simd_reduce(fx, v, Some(acc), ret, |fx, lane_layout, a, b| { if lane_layout.ty.is_floating_point() { fx.bcx.ins().fmul(a, b) } else { fx.bcx.ins().imul(a, b) } }); }; simd_reduce_all, (c v) { validate_simd_type!(fx, intrinsic, span, v.layout().ty); simd_reduce_bool(fx, v, ret, |fx, a, b| fx.bcx.ins().band(a, b)); }; simd_reduce_any, (c v) { validate_simd_type!(fx, intrinsic, span, v.layout().ty); simd_reduce_bool(fx, v, ret, |fx, a, b| fx.bcx.ins().bor(a, b)); }; simd_reduce_and, (c v) { validate_simd_type!(fx, intrinsic, span, v.layout().ty); simd_reduce(fx, v, None, ret, |fx, _layout, a, b| fx.bcx.ins().band(a, b)); }; simd_reduce_or, (c v) { validate_simd_type!(fx, intrinsic, span, v.layout().ty); simd_reduce(fx, v, None, ret, |fx, _layout, a, b| fx.bcx.ins().bor(a, b)); }; simd_reduce_xor, (c v) { validate_simd_type!(fx, intrinsic, span, v.layout().ty); simd_reduce(fx, v, None, ret, |fx, _layout, a, b| fx.bcx.ins().bxor(a, b)); }; simd_reduce_min, (c v) { // FIXME support floats validate_simd_type!(fx, intrinsic, span, v.layout().ty); simd_reduce(fx, v, None, ret, |fx, layout, a, b| { let lt = fx.bcx.ins().icmp(if layout.ty.is_signed() { IntCC::SignedLessThan } else { IntCC::UnsignedLessThan }, a, b); fx.bcx.ins().select(lt, a, b) }); }; simd_reduce_max, (c v) { // FIXME support floats validate_simd_type!(fx, intrinsic, span, v.layout().ty); simd_reduce(fx, v, None, ret, |fx, layout, a, b| { let gt = fx.bcx.ins().icmp(if layout.ty.is_signed() { IntCC::SignedGreaterThan } else { IntCC::UnsignedGreaterThan }, a, b); fx.bcx.ins().select(gt, a, b) }); }; simd_select, (c m, c a, c b) { validate_simd_type!(fx, intrinsic, span, m.layout().ty); validate_simd_type!(fx, intrinsic, span, a.layout().ty); assert_eq!(a.layout(), b.layout()); let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx); let lane_layout = fx.layout_of(lane_ty); for lane in 0..lane_count { let m_lane = m.value_lane(fx, lane).load_scalar(fx); let a_lane = a.value_lane(fx, lane).load_scalar(fx); let b_lane = b.value_lane(fx, lane).load_scalar(fx); let m_lane = fx.bcx.ins().icmp_imm(IntCC::Equal, m_lane, 0); let res_lane = CValue::by_val(fx.bcx.ins().select(m_lane, b_lane, a_lane), lane_layout); ret.place_lane(fx, lane).write_cvalue(fx, res_lane); } }; // simd_saturating_* // simd_bitmask // simd_scatter // simd_gather } }