Merge commit '1eded3619d0e55d57521a259bf27a03906fdfad0' into sync_cg_clif-2023-07-22

9 files changed, 352 insertions, 71 deletions

diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
index eddb479073c..2c038f22ca9 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
@@ -445,9 +445,14 @@ pub(crate) fn codegen_terminator_call<'tcx>(
 
     // Unpack arguments tuple for closures
     let mut args = if fn_sig.abi() == Abi::RustCall {
-        assert_eq!(args.len(), 2, "rust-call abi requires two arguments");
-        let self_arg = codegen_call_argument_operand(fx, &args[0]);
-        let pack_arg = codegen_call_argument_operand(fx, &args[1]);
+        let (self_arg, pack_arg) = match args {
+            [pack_arg] => (None, codegen_call_argument_operand(fx, pack_arg)),
+            [self_arg, pack_arg] => (
+                Some(codegen_call_argument_operand(fx, self_arg)),
+                codegen_call_argument_operand(fx, pack_arg),
+            ),
+            _ => panic!("rust-call abi requires one or two arguments"),
+        };
 
         let tupled_arguments = match pack_arg.value.layout().ty.kind() {
             ty::Tuple(ref tupled_arguments) => tupled_arguments,
@@ -455,7 +460,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
         };
 
         let mut args = Vec::with_capacity(1 + tupled_arguments.len());
-        args.push(self_arg);
+        args.extend(self_arg);
         for i in 0..tupled_arguments.len() {
             args.push(CallArgument {
                 value: pack_arg.value.value_field(fx, FieldIdx::new(i)),
diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs
index e05f2146f0c..522dd7189fe 100644
--- a/compiler/rustc_codegen_cranelift/src/base.rs
+++ b/compiler/rustc_codegen_cranelift/src/base.rs
@@ -7,6 +7,8 @@ use rustc_middle::ty::layout::FnAbiOf;
 use rustc_middle::ty::print::with_no_trimmed_paths;
 
 use cranelift_codegen::ir::UserFuncName;
+use cranelift_codegen::CodegenError;
+use cranelift_module::ModuleError;
 
 use crate::constant::ConstantCx;
 use crate::debuginfo::FunctionDebugContext;
@@ -172,7 +174,21 @@ pub(crate) fn compile_fn(
     // Define function
     cx.profiler.generic_activity("define function").run(|| {
         context.want_disasm = cx.should_write_ir;
-        module.define_function(codegened_func.func_id, context).unwrap();
+        match module.define_function(codegened_func.func_id, context) {
+            Ok(()) => {}
+            Err(ModuleError::Compilation(CodegenError::ImplLimitExceeded)) => {
+                let handler = rustc_session::EarlyErrorHandler::new(
+                    rustc_session::config::ErrorOutputType::default(),
+                );
+                handler.early_error(format!(
+                    "backend implementation limit exceeded while compiling {name}",
+                    name = codegened_func.symbol_name
+                ));
+            }
+            Err(err) => {
+                panic!("Error while defining {name}: {err:?}", name = codegened_func.symbol_name);
+            }
+        }
     });
 
     if cx.should_write_ir {
@@ -356,7 +372,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
 
                         codegen_panic_inner(
                             fx,
-                            rustc_hir::LangItem::PanicBoundsCheck,
+                            rustc_hir::LangItem::PanicMisalignedPointerDereference,
                             &[required, found, location],
                             source_info.span,
                         );
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
index 1b454b6667c..50bc7a127af 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
@@ -165,7 +165,7 @@ impl FunctionDebugContext {
         for &MachSrcLoc { start, end, loc } in mcr.buffer.get_srclocs_sorted() {
             debug_context.dwarf.unit.line_program.row().address_offset = u64::from(start);
             if !loc.is_default() {
-                let source_loc = *self.source_loc_set.get_index(loc.bits() as usize).unwrap();
+                let source_loc = self.source_loc_set[loc.bits() as usize];
                 create_row_for_span(debug_context, source_loc);
             } else {
                 create_row_for_span(debug_context, self.function_source_loc);
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
index 3a7421d8b30..8a4b1cccf14 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
@@ -38,7 +38,7 @@ pub(crate) struct DebugContext {
 pub(crate) struct FunctionDebugContext {
     entry_id: UnitEntryId,
     function_source_loc: (FileId, u64, u64),
-    source_loc_set: indexmap::IndexSet<(FileId, u64, u64)>,
+    source_loc_set: IndexSet<(FileId, u64, u64)>,
 }
 
 impl DebugContext {
diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
index 41e24acefbe..3ea38842148 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
@@ -114,9 +114,9 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
         .iter()
         .map(|cgu| cgu.items_in_deterministic_order(tcx).into_iter())
         .flatten()
-        .collect::<FxHashMap<_, (_, _)>>()
+        .collect::<FxHashMap<_, _>>()
         .into_iter()
-        .collect::<Vec<(_, (_, _))>>();
+        .collect::<Vec<(_, _)>>();
 
     tcx.sess.time("codegen mono items", || {
         super::predefine_mono_items(tcx, &mut jit_module, &mono_items);
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
index 18162fb5ab2..fdd27a454e0 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
@@ -18,6 +18,20 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             // Spin loop hint
         }
 
+        // Used by is_x86_feature_detected!();
+        "llvm.x86.xgetbv" => {
+            // FIXME use the actual xgetbv instruction
+            intrinsic_args!(fx, args => (v); intrinsic);
+
+            let v = v.load_scalar(fx);
+
+            // As of writing on XCR0 exists
+            fx.bcx.ins().trapnz(v, TrapCode::UnreachableCodeReached);
+
+            let res = fx.bcx.ins().iconst(types::I64, 1 /* bit 0 must be set */);
+            ret.write_cvalue(fx, CValue::by_val(res, fx.layout_of(fx.tcx.types.i64)));
+        }
+
         // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8`
         "llvm.x86.sse2.pmovmskb.128"
         | "llvm.x86.avx2.pmovmskb"
@@ -53,7 +67,7 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             let res = CValue::by_val(res, fx.layout_of(fx.tcx.types.i32));
             ret.write_cvalue(fx, res);
         }
-        "llvm.x86.sse2.cmp.ps" | "llvm.x86.sse2.cmp.pd" => {
+        "llvm.x86.sse.cmp.ps" | "llvm.x86.sse2.cmp.pd" => {
             let (x, y, kind) = match args {
                 [x, y, kind] => (x, y, kind),
                 _ => bug!("wrong number of args for intrinsic {intrinsic}"),
@@ -66,18 +80,95 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             let flt_cc = match kind
                 .try_to_bits(Size::from_bytes(1))
                 .unwrap_or_else(|| panic!("kind not scalar: {:?}", kind))
+                .try_into()
+                .unwrap()
             {
-                0 => FloatCC::Equal,
-                1 => FloatCC::LessThan,
-                2 => FloatCC::LessThanOrEqual,
-                7 => FloatCC::Ordered,
-                3 => FloatCC::Unordered,
-                4 => FloatCC::NotEqual,
-                5 => FloatCC::UnorderedOrGreaterThanOrEqual,
-                6 => FloatCC::UnorderedOrGreaterThan,
+                _CMP_EQ_OQ | _CMP_EQ_OS => FloatCC::Equal,
+                _CMP_LT_OS | _CMP_LT_OQ => FloatCC::LessThan,
+                _CMP_LE_OS | _CMP_LE_OQ => FloatCC::LessThanOrEqual,
+                _CMP_UNORD_Q | _CMP_UNORD_S => FloatCC::Unordered,
+                _CMP_NEQ_UQ | _CMP_NEQ_US => FloatCC::NotEqual,
+                _CMP_NLT_US | _CMP_NLT_UQ => FloatCC::UnorderedOrGreaterThanOrEqual,
+                _CMP_NLE_US | _CMP_NLE_UQ => FloatCC::UnorderedOrGreaterThan,
+                _CMP_ORD_Q | _CMP_ORD_S => FloatCC::Ordered,
+                _CMP_EQ_UQ | _CMP_EQ_US => FloatCC::UnorderedOrEqual,
+                _CMP_NGE_US | _CMP_NGE_UQ => FloatCC::UnorderedOrLessThan,
+                _CMP_NGT_US | _CMP_NGT_UQ => FloatCC::UnorderedOrLessThanOrEqual,
+                _CMP_FALSE_OQ | _CMP_FALSE_OS => todo!(),
+                _CMP_NEQ_OQ | _CMP_NEQ_OS => FloatCC::OrderedNotEqual,
+                _CMP_GE_OS | _CMP_GE_OQ => FloatCC::GreaterThanOrEqual,
+                _CMP_GT_OS | _CMP_GT_OQ => FloatCC::GreaterThan,
+                _CMP_TRUE_UQ | _CMP_TRUE_US => todo!(),
+
                 kind => unreachable!("kind {:?}", kind),
             };
 
+            // Copied from stdarch
+            /// Equal (ordered, non-signaling)
+            const _CMP_EQ_OQ: i32 = 0x00;
+            /// Less-than (ordered, signaling)
+            const _CMP_LT_OS: i32 = 0x01;
+            /// Less-than-or-equal (ordered, signaling)
+            const _CMP_LE_OS: i32 = 0x02;
+            /// Unordered (non-signaling)
+            const _CMP_UNORD_Q: i32 = 0x03;
+            /// Not-equal (unordered, non-signaling)
+            const _CMP_NEQ_UQ: i32 = 0x04;
+            /// Not-less-than (unordered, signaling)
+            const _CMP_NLT_US: i32 = 0x05;
+            /// Not-less-than-or-equal (unordered, signaling)
+            const _CMP_NLE_US: i32 = 0x06;
+            /// Ordered (non-signaling)
+            const _CMP_ORD_Q: i32 = 0x07;
+            /// Equal (unordered, non-signaling)
+            const _CMP_EQ_UQ: i32 = 0x08;
+            /// Not-greater-than-or-equal (unordered, signaling)
+            const _CMP_NGE_US: i32 = 0x09;
+            /// Not-greater-than (unordered, signaling)
+            const _CMP_NGT_US: i32 = 0x0a;
+            /// False (ordered, non-signaling)
+            const _CMP_FALSE_OQ: i32 = 0x0b;
+            /// Not-equal (ordered, non-signaling)
+            const _CMP_NEQ_OQ: i32 = 0x0c;
+            /// Greater-than-or-equal (ordered, signaling)
+            const _CMP_GE_OS: i32 = 0x0d;
+            /// Greater-than (ordered, signaling)
+            const _CMP_GT_OS: i32 = 0x0e;
+            /// True (unordered, non-signaling)
+            const _CMP_TRUE_UQ: i32 = 0x0f;
+            /// Equal (ordered, signaling)
+            const _CMP_EQ_OS: i32 = 0x10;
+            /// Less-than (ordered, non-signaling)
+            const _CMP_LT_OQ: i32 = 0x11;
+            /// Less-than-or-equal (ordered, non-signaling)
+            const _CMP_LE_OQ: i32 = 0x12;
+            /// Unordered (signaling)
+            const _CMP_UNORD_S: i32 = 0x13;
+            /// Not-equal (unordered, signaling)
+            const _CMP_NEQ_US: i32 = 0x14;
+            /// Not-less-than (unordered, non-signaling)
+            const _CMP_NLT_UQ: i32 = 0x15;
+            /// Not-less-than-or-equal (unordered, non-signaling)
+            const _CMP_NLE_UQ: i32 = 0x16;
+            /// Ordered (signaling)
+            const _CMP_ORD_S: i32 = 0x17;
+            /// Equal (unordered, signaling)
+            const _CMP_EQ_US: i32 = 0x18;
+            /// Not-greater-than-or-equal (unordered, non-signaling)
+            const _CMP_NGE_UQ: i32 = 0x19;
+            /// Not-greater-than (unordered, non-signaling)
+            const _CMP_NGT_UQ: i32 = 0x1a;
+            /// False (ordered, signaling)
+            const _CMP_FALSE_OS: i32 = 0x1b;
+            /// Not-equal (ordered, signaling)
+            const _CMP_NEQ_OS: i32 = 0x1c;
+            /// Greater-than-or-equal (ordered, non-signaling)
+            const _CMP_GE_OQ: i32 = 0x1d;
+            /// Greater-than (ordered, non-signaling)
+            const _CMP_GT_OQ: i32 = 0x1e;
+            /// True (unordered, signaling)
+            const _CMP_TRUE_US: i32 = 0x1f;
+
             simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| {
                 let res_lane = match lane_ty.kind() {
                     ty::Float(_) => fx.bcx.ins().fcmp(flt_cc, x_lane, y_lane),
@@ -103,6 +194,23 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
                 _ => fx.bcx.ins().iconst(types::I32, 0),
             });
         }
+        "llvm.x86.sse2.psrai.d" => {
+            let (a, imm8) = match args {
+                [a, imm8] => (a, imm8),
+                _ => bug!("wrong number of args for intrinsic {intrinsic}"),
+            };
+            let a = codegen_operand(fx, a);
+            let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8)
+                .expect("llvm.x86.sse2.psrai.d imm8 not const");
+
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8
+                .try_to_bits(Size::from_bytes(4))
+                .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8))
+            {
+                imm8 if imm8 < 32 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)),
+                _ => fx.bcx.ins().iconst(types::I32, 0),
+            });
+        }
         "llvm.x86.sse2.pslli.d" => {
             let (a, imm8) = match args {
                 [a, imm8] => (a, imm8),
@@ -137,6 +245,23 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
                 _ => fx.bcx.ins().iconst(types::I32, 0),
             });
         }
+        "llvm.x86.sse2.psrai.w" => {
+            let (a, imm8) = match args {
+                [a, imm8] => (a, imm8),
+                _ => bug!("wrong number of args for intrinsic {intrinsic}"),
+            };
+            let a = codegen_operand(fx, a);
+            let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8)
+                .expect("llvm.x86.sse2.psrai.d imm8 not const");
+
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8
+                .try_to_bits(Size::from_bytes(4))
+                .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8))
+            {
+                imm8 if imm8 < 16 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)),
+                _ => fx.bcx.ins().iconst(types::I32, 0),
+            });
+        }
         "llvm.x86.sse2.pslli.w" => {
             let (a, imm8) = match args {
                 [a, imm8] => (a, imm8),
@@ -171,6 +296,57 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
                 _ => fx.bcx.ins().iconst(types::I32, 0),
             });
         }
+        "llvm.x86.avx.psrai.d" => {
+            let (a, imm8) = match args {
+                [a, imm8] => (a, imm8),
+                _ => bug!("wrong number of args for intrinsic {intrinsic}"),
+            };
+            let a = codegen_operand(fx, a);
+            let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8)
+                .expect("llvm.x86.avx.psrai.d imm8 not const");
+
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8
+                .try_to_bits(Size::from_bytes(4))
+                .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8))
+            {
+                imm8 if imm8 < 32 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)),
+                _ => fx.bcx.ins().iconst(types::I32, 0),
+            });
+        }
+        "llvm.x86.sse2.psrli.q" => {
+            let (a, imm8) = match args {
+                [a, imm8] => (a, imm8),
+                _ => bug!("wrong number of args for intrinsic {intrinsic}"),
+            };
+            let a = codegen_operand(fx, a);
+            let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8)
+                .expect("llvm.x86.avx.psrli.q imm8 not const");
+
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8
+                .try_to_bits(Size::from_bytes(4))
+                .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8))
+            {
+                imm8 if imm8 < 64 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)),
+                _ => fx.bcx.ins().iconst(types::I32, 0),
+            });
+        }
+        "llvm.x86.sse2.pslli.q" => {
+            let (a, imm8) = match args {
+                [a, imm8] => (a, imm8),
+                _ => bug!("wrong number of args for intrinsic {intrinsic}"),
+            };
+            let a = codegen_operand(fx, a);
+            let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8)
+                .expect("llvm.x86.avx.pslli.q imm8 not const");
+
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8
+                .try_to_bits(Size::from_bytes(4))
+                .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8))
+            {
+                imm8 if imm8 < 64 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)),
+                _ => fx.bcx.ins().iconst(types::I32, 0),
+            });
+        }
         "llvm.x86.avx.pslli.d" => {
             let (a, imm8) = match args {
                 [a, imm8] => (a, imm8),
@@ -205,6 +381,23 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
                 _ => fx.bcx.ins().iconst(types::I32, 0),
             });
         }
+        "llvm.x86.avx2.psrai.w" => {
+            let (a, imm8) = match args {
+                [a, imm8] => (a, imm8),
+                _ => bug!("wrong number of args for intrinsic {intrinsic}"),
+            };
+            let a = codegen_operand(fx, a);
+            let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8)
+                .expect("llvm.x86.avx.psrai.w imm8 not const");
+
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8
+                .try_to_bits(Size::from_bytes(4))
+                .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8))
+            {
+                imm8 if imm8 < 16 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)),
+                _ => fx.bcx.ins().iconst(types::I32, 0),
+            });
+        }
         "llvm.x86.avx2.pslli.w" => {
             let (a, imm8) = match args {
                 [a, imm8] => (a, imm8),
@@ -313,7 +506,7 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             ret.place_lane(fx, 2).to_ptr().store(fx, res_2, MemFlags::trusted());
             ret.place_lane(fx, 3).to_ptr().store(fx, res_3, MemFlags::trusted());
         }
-        "llvm.x86.sse2.storeu.dq" => {
+        "llvm.x86.sse2.storeu.dq" | "llvm.x86.sse2.storeu.pd" => {
             intrinsic_args!(fx, args => (mem_addr, a); intrinsic);
             let mem_addr = mem_addr.load_scalar(fx);
 
@@ -321,17 +514,45 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             let dest = CPlace::for_ptr(Pointer::new(mem_addr), a.layout());
             dest.write_cvalue(fx, a);
         }
-        "llvm.x86.addcarry.64" => {
+        "llvm.x86.ssse3.pabs.b.128" | "llvm.x86.ssse3.pabs.w.128" | "llvm.x86.ssse3.pabs.d.128" => {
+            let a = match args {
+                [a] => a,
+                _ => bug!("wrong number of args for intrinsic {intrinsic}"),
+            };
+            let a = codegen_operand(fx, a);
+
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| {
+                fx.bcx.ins().iabs(lane)
+            });
+        }
+        "llvm.x86.addcarry.32" | "llvm.x86.addcarry.64" => {
             intrinsic_args!(fx, args => (c_in, a, b); intrinsic);
             let c_in = c_in.load_scalar(fx);
 
-            llvm_add_sub(fx, BinOp::Add, ret, c_in, a, b);
+            let (cb_out, c) = llvm_add_sub(fx, BinOp::Add, c_in, a, b);
+
+            let layout = fx.layout_of(Ty::new_tup(fx.tcx, &[fx.tcx.types.u8, a.layout().ty]));
+            let val = CValue::by_val_pair(cb_out, c, layout);
+            ret.write_cvalue(fx, val);
         }
-        "llvm.x86.subborrow.64" => {
+        "llvm.x86.addcarryx.u32" | "llvm.x86.addcarryx.u64" => {
+            intrinsic_args!(fx, args => (c_in, a, b, out); intrinsic);
+            let c_in = c_in.load_scalar(fx);
+
+            let (cb_out, c) = llvm_add_sub(fx, BinOp::Add, c_in, a, b);
+
+            Pointer::new(out.load_scalar(fx)).store(fx, c, MemFlags::trusted());
+            ret.write_cvalue(fx, CValue::by_val(cb_out, fx.layout_of(fx.tcx.types.u8)));
+        }
+        "llvm.x86.subborrow.32" | "llvm.x86.subborrow.64" => {
             intrinsic_args!(fx, args => (b_in, a, b); intrinsic);
             let b_in = b_in.load_scalar(fx);
 
-            llvm_add_sub(fx, BinOp::Sub, ret, b_in, a, b);
+            let (cb_out, c) = llvm_add_sub(fx, BinOp::Sub, b_in, a, b);
+
+            let layout = fx.layout_of(Ty::new_tup(fx.tcx, &[fx.tcx.types.u8, a.layout().ty]));
+            let val = CValue::by_val_pair(cb_out, c, layout);
+            ret.write_cvalue(fx, val);
         }
         _ => {
             fx.tcx
@@ -356,21 +577,11 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
 fn llvm_add_sub<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     bin_op: BinOp,
-    ret: CPlace<'tcx>,
     cb_in: Value,
     a: CValue<'tcx>,
     b: CValue<'tcx>,
-) {
-    assert_eq!(
-        a.layout().ty,
-        fx.tcx.types.u64,
-        "llvm.x86.addcarry.64/llvm.x86.subborrow.64 second operand must be u64"
-    );
-    assert_eq!(
-        b.layout().ty,
-        fx.tcx.types.u64,
-        "llvm.x86.addcarry.64/llvm.x86.subborrow.64 third operand must be u64"
-    );
+) -> (Value, Value) {
+    assert_eq!(a.layout().ty, b.layout().ty);
 
     // c + carry -> c + first intermediate carry or borrow respectively
     let int0 = crate::num::codegen_checked_int_binop(fx, bin_op, a, b);
@@ -378,15 +589,14 @@ fn llvm_add_sub<'tcx>(
     let cb0 = int0.value_field(fx, FieldIdx::new(1)).load_scalar(fx);
 
     // c + carry -> c + second intermediate carry or borrow respectively
-    let cb_in_as_u64 = fx.bcx.ins().uextend(types::I64, cb_in);
-    let cb_in_as_u64 = CValue::by_val(cb_in_as_u64, fx.layout_of(fx.tcx.types.u64));
-    let int1 = crate::num::codegen_checked_int_binop(fx, bin_op, c, cb_in_as_u64);
+    let clif_ty = fx.clif_type(a.layout().ty).unwrap();
+    let cb_in_as_int = fx.bcx.ins().uextend(clif_ty, cb_in);
+    let cb_in_as_int = CValue::by_val(cb_in_as_int, fx.layout_of(a.layout().ty));
+    let int1 = crate::num::codegen_checked_int_binop(fx, bin_op, c, cb_in_as_int);
     let (c, cb1) = int1.load_scalar_pair(fx);
 
     // carry0 | carry1 -> carry or borrow respectively
     let cb_out = fx.bcx.ins().bor(cb0, cb1);
 
-    let layout = fx.layout_of(Ty::new_tup(fx.tcx, &[fx.tcx.types.u8, fx.tcx.types.u64]));
-    let val = CValue::by_val_pair(cb_out, c, layout);
-    ret.write_cvalue(fx, val);
+    (cb_out, c)
 }
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
index da8ab361331..e3006b253b7 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
@@ -647,12 +647,13 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
             ret.write_cvalue(fx, val);
         }
-        sym::volatile_store | sym::unaligned_volatile_store => {
+        sym::volatile_store | sym::unaligned_volatile_store | sym::nontemporal_store => {
             intrinsic_args!(fx, args => (ptr, val); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
             // Cranelift treats stores as volatile by default
             // FIXME correctly handle unaligned_volatile_store
+            // FIXME actually do nontemporal stores if requested
             let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
             dest.write_cvalue(fx, val);
         }
diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs
index 0de2dccda71..ebd153cb71d 100644
--- a/compiler/rustc_codegen_cranelift/src/lib.rs
+++ b/compiler/rustc_codegen_cranelift/src/lib.rs
@@ -268,8 +268,6 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Arc<dyn isa::Tar
     };
     flags_builder.set("tls_model", tls_model).unwrap();
 
-    flags_builder.set("enable_simd", "true").unwrap();
-
     flags_builder.set("enable_llvm_abi_extensions", "true").unwrap();
 
     use rustc_session::config::OptLevel;
diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
index ec0b61a7ce5..ff95141ce90 100644
--- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs
+++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
@@ -2,6 +2,8 @@
 
 use crate::prelude::*;
 
+use rustc_middle::ty::FnSig;
+
 use cranelift_codegen::entity::EntityRef;
 use cranelift_codegen::ir::immediates::Offset32;
 
@@ -160,6 +162,7 @@ impl<'tcx> CValue<'tcx> {
     }
 
     /// Load a value with layout.abi of scalar
+    #[track_caller]
     pub(crate) fn load_scalar(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> Value {
         let layout = self.1;
         match self.0 {
@@ -182,6 +185,7 @@ impl<'tcx> CValue<'tcx> {
     }
 
     /// Load a value pair with layout.abi of scalar pair
+    #[track_caller]
     pub(crate) fn load_scalar_pair(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> (Value, Value) {
         let layout = self.1;
         match self.0 {
@@ -583,17 +587,25 @@ impl<'tcx> CPlace<'tcx> {
         let dst_layout = self.layout();
         match self.inner {
             CPlaceInner::Var(_local, var) => {
-                let data = CValue(from.0, dst_layout).load_scalar(fx);
+                let data = match from.1.abi {
+                    Abi::Scalar(_) => CValue(from.0, dst_layout).load_scalar(fx),
+                    _ => {
+                        let (ptr, meta) = from.force_stack(fx);
+                        assert!(meta.is_none());
+                        CValue(CValueInner::ByRef(ptr, None), dst_layout).load_scalar(fx)
+                    }
+                };
                 let dst_ty = fx.clif_type(self.layout().ty).unwrap();
                 transmute_scalar(fx, var, data, dst_ty);
             }
             CPlaceInner::VarPair(_local, var1, var2) => {
-                let (data1, data2) = if from.layout().ty == dst_layout.ty {
-                    CValue(from.0, dst_layout).load_scalar_pair(fx)
-                } else {
-                    let (ptr, meta) = from.force_stack(fx);
-                    assert!(meta.is_none());
-                    CValue(CValueInner::ByRef(ptr, None), dst_layout).load_scalar_pair(fx)
+                let (data1, data2) = match from.1.abi {
+                    Abi::ScalarPair(_, _) => CValue(from.0, dst_layout).load_scalar_pair(fx),
+                    _ => {
+                        let (ptr, meta) = from.force_stack(fx);
+                        assert!(meta.is_none());
+                        CValue(CValueInner::ByRef(ptr, None), dst_layout).load_scalar_pair(fx)
+                    }
                 };
                 let (dst_ty1, dst_ty2) = fx.clif_pair_type(self.layout().ty).unwrap();
                 transmute_scalar(fx, var1, data1, dst_ty1);
@@ -607,30 +619,38 @@ impl<'tcx> CPlace<'tcx> {
 
                 let mut flags = MemFlags::new();
                 flags.set_notrap();
-                match from.layout().abi {
-                    Abi::Scalar(_) => {
-                        let val = from.load_scalar(fx);
-                        to_ptr.store(fx, val, flags);
-                        return;
-                    }
-                    Abi::ScalarPair(a_scalar, b_scalar) => {
-                        let (value, extra) = from.load_scalar_pair(fx);
-                        let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar);
-                        to_ptr.store(fx, value, flags);
-                        to_ptr.offset(fx, b_offset).store(fx, extra, flags);
-                        return;
-                    }
-                    _ => {}
-                }
 
                 match from.0 {
                     CValueInner::ByVal(val) => {
                         to_ptr.store(fx, val, flags);
                     }
-                    CValueInner::ByValPair(_, _) => {
-                        bug!("Non ScalarPair abi {:?} for ByValPair CValue", dst_layout.abi);
-                    }
+                    CValueInner::ByValPair(val1, val2) => match from.layout().abi {
+                        Abi::ScalarPair(a_scalar, b_scalar) => {
+                            let b_offset =
+                                scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar);
+                            to_ptr.store(fx, val1, flags);
+                            to_ptr.offset(fx, b_offset).store(fx, val2, flags);
+                        }
+                        _ => bug!("Non ScalarPair abi {:?} for ByValPair CValue", dst_layout.abi),
+                    },
                     CValueInner::ByRef(from_ptr, None) => {
+                        match from.layout().abi {
+                            Abi::Scalar(_) => {
+                                let val = from.load_scalar(fx);
+                                to_ptr.store(fx, val, flags);
+                                return;
+                            }
+                            Abi::ScalarPair(a_scalar, b_scalar) => {
+                                let b_offset =
+                                    scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar);
+                                let (val1, val2) = from.load_scalar_pair(fx);
+                                to_ptr.store(fx, val1, flags);
+                                to_ptr.offset(fx, b_offset).store(fx, val2, flags);
+                                return;
+                            }
+                            _ => {}
+                        }
+
                         let from_addr = from_ptr.get_addr(fx);
                         let to_addr = to_ptr.get_addr(fx);
                         let src_layout = from.1;
@@ -815,11 +835,42 @@ pub(crate) fn assert_assignable<'tcx>(
                 ParamEnv::reveal_all(),
                 from_ty.fn_sig(fx.tcx),
             );
+            let FnSig {
+                inputs_and_output: types_from,
+                c_variadic: c_variadic_from,
+                unsafety: unsafety_from,
+                abi: abi_from,
+            } = from_sig;
             let to_sig = fx
                 .tcx
                 .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to_ty.fn_sig(fx.tcx));
+            let FnSig {
+                inputs_and_output: types_to,
+                c_variadic: c_variadic_to,
+                unsafety: unsafety_to,
+                abi: abi_to,
+            } = to_sig;
+            let mut types_from = types_from.iter();
+            let mut types_to = types_to.iter();
+            loop {
+                match (types_from.next(), types_to.next()) {
+                    (Some(a), Some(b)) => assert_assignable(fx, a, b, limit - 1),
+                    (None, None) => break,
+                    (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty),
+                }
+            }
+            assert_eq!(
+                c_variadic_from, c_variadic_to,
+                "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}",
+                from_sig, to_sig, fx,
+            );
+            assert_eq!(
+                unsafety_from, unsafety_to,
+                "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}",
+                from_sig, to_sig, fx,
+            );
             assert_eq!(
-                from_sig, to_sig,
+                abi_from, abi_to,
                 "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}",
                 from_sig, to_sig, fx,
             );