diff options
Diffstat (limited to 'compiler/rustc_codegen_cranelift/src')
5 files changed, 66 insertions, 28 deletions
diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs index cb74c59046b..892ec3e9585 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs @@ -508,7 +508,10 @@ pub(crate) fn codegen_terminator_call<'tcx>( let nop_inst = fx.bcx.ins().nop(); fx.add_comment( nop_inst, - format!("virtual call; self arg pass mode: {:?}", fn_abi.args[0]), + with_no_trimmed_paths!(format!( + "virtual call; self arg pass mode: {:?}", + fn_abi.args[0] + )), ); } diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index 8964230d6e2..6ba6a64c544 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -490,6 +490,11 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant } /// Used only for intrinsic implementations that need a compile-time constant +/// +/// All uses of this function are a bug inside stdarch. [`eval_mir_constant`] +/// should be used everywhere, but for some vendor intrinsics stdarch forgets +/// to wrap the immediate argument in `const {}`, necesitating this hack to get +/// the correct value at compile time instead. pub(crate) fn mir_operand_get_const_val<'tcx>( fx: &FunctionCx<'_, '_, 'tcx>, operand: &Operand<'tcx>, diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs index f0fb18608e0..39f6763d9f2 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs @@ -91,6 +91,44 @@ pub(crate) fn codegen_aarch64_llvm_intrinsic_call<'tcx>( ); } + _ if intrinsic.starts_with("llvm.aarch64.neon.fmax.v") => { + intrinsic_args!(fx, args => (x, y); intrinsic); + + simd_pair_for_each_lane( + fx, + x, + y, + ret, + &|fx, _lane_ty, _res_lane_ty, x_lane, y_lane| fx.bcx.ins().fmax(x_lane, y_lane), + ); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.fmin.v") => { + intrinsic_args!(fx, args => (x, y); intrinsic); + + simd_pair_for_each_lane( + fx, + x, + y, + ret, + &|fx, _lane_ty, _res_lane_ty, x_lane, y_lane| fx.bcx.ins().fmin(x_lane, y_lane), + ); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.faddv.f32.v") => { + intrinsic_args!(fx, args => (v); intrinsic); + + simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().fadd(a, b)); + } + + _ if intrinsic.starts_with("llvm.aarch64.neon.frintn.v") => { + intrinsic_args!(fx, args => (v); intrinsic); + + simd_for_each_lane(fx, v, ret, &|fx, _lane_ty, _res_lane_ty, lane| { + fx.bcx.ins().nearest(lane) + }); + } + _ if intrinsic.starts_with("llvm.aarch64.neon.smaxv.i") => { intrinsic_args!(fx, args => (v); intrinsic); diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index 0740c84a534..19e5adc2538 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -600,9 +600,11 @@ fn codegen_regular_intrinsic_call<'tcx>( sym::ptr_mask => { intrinsic_args!(fx, args => (ptr, mask); intrinsic); + let ptr_layout = ptr.layout(); let ptr = ptr.load_scalar(fx); let mask = mask.load_scalar(fx); - fx.bcx.ins().band(ptr, mask); + let res = fx.bcx.ins().band(ptr, mask); + ret.write_cvalue(fx, CValue::by_val(res, ptr_layout)); } sym::write_bytes | sym::volatile_set_memory => { diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs index f2bdf24d8ab..ff4b0e847af 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs @@ -181,11 +181,9 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( // Make sure this is actually a SIMD vector. let idx_ty = fx.monomorphize(idx.node.ty(fx.mir, fx.tcx)); - let n: u16 = if idx_ty.is_simd() - && matches!(idx_ty.simd_size_and_type(fx.tcx).1.kind(), ty::Uint(ty::UintTy::U32)) + if !idx_ty.is_simd() + || !matches!(idx_ty.simd_size_and_type(fx.tcx).1.kind(), ty::Uint(ty::UintTy::U32)) { - idx_ty.simd_size_and_type(fx.tcx).0.try_into().unwrap() - } else { fx.tcx.dcx().span_err( span, format!("simd_shuffle index must be a SIMD vector of `u32`, got `{}`", idx_ty), @@ -194,6 +192,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); return; }; + let n: u16 = idx_ty.simd_size_and_type(fx.tcx).0.try_into().unwrap(); assert_eq!(x.layout(), y.layout()); let layout = x.layout(); @@ -268,10 +267,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let val = codegen_operand(fx, &val.node); // FIXME validate - let idx_const = if let Some(idx_const) = - crate::constant::mir_operand_get_const_val(fx, &idx.node) - { - idx_const + let idx_const = if let Some(idx_const) = idx.node.constant() { + crate::constant::eval_mir_constant(fx, idx_const).0.try_to_scalar_int().unwrap() } else { fx.tcx.dcx().span_fatal(span, "Index argument for `simd_insert` is not a constant"); }; @@ -304,22 +301,12 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( return; } - let idx_const = if let Some(idx_const) = - crate::constant::mir_operand_get_const_val(fx, &idx.node) - { - idx_const + let idx_const = if let Some(idx_const) = idx.node.constant() { + crate::constant::eval_mir_constant(fx, idx_const).0.try_to_scalar_int().unwrap() } else { - fx.tcx.dcx().span_warn(span, "Index argument for `simd_extract` is not a constant"); - let trap_block = fx.bcx.create_block(); - let true_ = fx.bcx.ins().iconst(types::I8, 1); - let ret_block = fx.get_block(target); - fx.bcx.ins().brif(true_, trap_block, &[], ret_block, &[]); - fx.bcx.switch_to_block(trap_block); - crate::trap::trap_unimplemented( - fx, - "Index argument for `simd_extract` is not a constant", - ); - return; + fx.tcx + .dcx() + .span_fatal(span, "Index argument for `simd_extract` is not a constant"); }; let idx = idx_const.to_u32(); @@ -574,9 +561,12 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( (sym::simd_round, types::F64) => "round", _ => unreachable!("{:?}", intrinsic), }; - fx.lib_call(name, vec![AbiParam::new(lane_ty)], vec![AbiParam::new(lane_ty)], &[ - lane, - ])[0] + fx.lib_call( + name, + vec![AbiParam::new(lane_ty)], + vec![AbiParam::new(lane_ty)], + &[lane], + )[0] }); } |