Merge commit '63734fcdd718cca089f84c42f3a42c0096cfd431' into sync_cg_clif-2022-05-15

8 files changed, 109 insertions, 253 deletions

diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
index ef56fb191bf..b163a426191 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
@@ -309,7 +309,7 @@ fn codegen_call_argument_operand<'tcx>(
 
 pub(crate) fn codegen_terminator_call<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
-    span: Span,
+    source_info: mir::SourceInfo,
     func: &Operand<'tcx>,
     args: &[Operand<'tcx>],
     mir_dest: Option<(Place<'tcx>, BasicBlock)>,
@@ -340,7 +340,13 @@ pub(crate) fn codegen_terminator_call<'tcx>(
 
         match instance.def {
             InstanceDef::Intrinsic(_) => {
-                crate::intrinsics::codegen_intrinsic_call(fx, instance, args, destination, span);
+                crate::intrinsics::codegen_intrinsic_call(
+                    fx,
+                    instance,
+                    args,
+                    destination,
+                    source_info,
+                );
                 return;
             }
             InstanceDef::DropGlue(_, None) => {
@@ -402,7 +408,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
 
     // Pass the caller location for `#[track_caller]`.
     if instance.map(|inst| inst.def.requires_caller_location(fx.tcx)).unwrap_or(false) {
-        let caller_location = fx.get_caller_location(span);
+        let caller_location = fx.get_caller_location(source_info);
         args.push(CallArgument { value: caller_location, is_owned: false });
     }
 
@@ -479,9 +485,10 @@ pub(crate) fn codegen_terminator_call<'tcx>(
         // FIXME find a cleaner way to support varargs
         if fn_sig.c_variadic {
             if !matches!(fn_sig.abi, Abi::C { .. }) {
-                fx.tcx
-                    .sess
-                    .span_fatal(span, &format!("Variadic call for non-C abi {:?}", fn_sig.abi));
+                fx.tcx.sess.span_fatal(
+                    source_info.span,
+                    &format!("Variadic call for non-C abi {:?}", fn_sig.abi),
+                );
             }
             let sig_ref = fx.bcx.func.dfg.call_signature(call_inst).unwrap();
             let abi_params = call_args
@@ -490,9 +497,10 @@ pub(crate) fn codegen_terminator_call<'tcx>(
                     let ty = fx.bcx.func.dfg.value_type(arg);
                     if !ty.is_int() {
                         // FIXME set %al to upperbound on float args once floats are supported
-                        fx.tcx
-                            .sess
-                            .span_fatal(span, &format!("Non int ty {:?} for variadic call", ty));
+                        fx.tcx.sess.span_fatal(
+                            source_info.span,
+                            &format!("Non int ty {:?} for variadic call", ty),
+                        );
                     }
                     AbiParam::new(ty)
                 })
@@ -513,7 +521,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
 
 pub(crate) fn codegen_drop<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
-    span: Span,
+    source_info: mir::SourceInfo,
     drop_place: CPlace<'tcx>,
 ) {
     let ty = drop_place.layout().ty;
@@ -560,7 +568,7 @@ pub(crate) fn codegen_drop<'tcx>(
 
                 if drop_instance.def.requires_caller_location(fx.tcx) {
                     // Pass the caller location for `#[track_caller]`.
-                    let caller_location = fx.get_caller_location(span);
+                    let caller_location = fx.get_caller_location(source_info);
                     call_args.extend(
                         adjust_arg_for_abi(fx, caller_location, &fn_abi.args[1], false).into_iter(),
                     );
diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs
index 65346cb3962..65e5812a8a5 100644
--- a/compiler/rustc_codegen_cranelift/src/base.rs
+++ b/compiler/rustc_codegen_cranelift/src/base.rs
@@ -325,7 +325,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
                     AssertKind::BoundsCheck { ref len, ref index } => {
                         let len = codegen_operand(fx, len).load_scalar(fx);
                         let index = codegen_operand(fx, index).load_scalar(fx);
-                        let location = fx.get_caller_location(source_info.span).load_scalar(fx);
+                        let location = fx.get_caller_location(source_info).load_scalar(fx);
 
                         codegen_panic_inner(
                             fx,
@@ -336,7 +336,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
                     }
                     _ => {
                         let msg_str = msg.description();
-                        codegen_panic(fx, msg_str, source_info.span);
+                        codegen_panic(fx, msg_str, source_info);
                     }
                 }
             }
@@ -398,7 +398,13 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
                 from_hir_call: _,
             } => {
                 fx.tcx.sess.time("codegen call", || {
-                    crate::abi::codegen_terminator_call(fx, *fn_span, func, args, *destination)
+                    crate::abi::codegen_terminator_call(
+                        fx,
+                        mir::SourceInfo { span: *fn_span, ..source_info },
+                        func,
+                        args,
+                        *destination,
+                    )
                 });
             }
             TerminatorKind::InlineAsm {
@@ -450,7 +456,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
             }
             TerminatorKind::Drop { place, target, unwind: _ } => {
                 let drop_place = codegen_place(fx, *place);
-                crate::abi::codegen_drop(fx, source_info.span, drop_place);
+                crate::abi::codegen_drop(fx, source_info, drop_place);
 
                 let target_block = fx.get_block(*target);
                 fx.bcx.ins().jump(target_block, &[]);
@@ -471,7 +477,7 @@ fn codegen_stmt<'tcx>(
 
     fx.set_debug_loc(stmt.source_info);
 
-    #[cfg(disabled)]
+    #[cfg(any())] // This is never true
     match &stmt.kind {
         StatementKind::StorageLive(..) | StatementKind::StorageDead(..) => {} // Those are not very useful
         _ => {
@@ -898,14 +904,18 @@ pub(crate) fn codegen_operand<'tcx>(
     }
 }
 
-pub(crate) fn codegen_panic<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, msg_str: &str, span: Span) {
-    let location = fx.get_caller_location(span).load_scalar(fx);
+pub(crate) fn codegen_panic<'tcx>(
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
+    msg_str: &str,
+    source_info: mir::SourceInfo,
+) {
+    let location = fx.get_caller_location(source_info).load_scalar(fx);
 
     let msg_ptr = fx.anonymous_str(msg_str);
     let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap());
     let args = [msg_ptr, msg_len, location];
 
-    codegen_panic_inner(fx, rustc_hir::LangItem::Panic, &args, span);
+    codegen_panic_inner(fx, rustc_hir::LangItem::Panic, &args, source_info.span);
 }
 
 pub(crate) fn codegen_panic_inner<'tcx>(
diff --git a/compiler/rustc_codegen_cranelift/src/bin/cg_clif.rs b/compiler/rustc_codegen_cranelift/src/bin/cg_clif.rs
deleted file mode 100644
index 5984ec8412a..00000000000
--- a/compiler/rustc_codegen_cranelift/src/bin/cg_clif.rs
+++ /dev/null
@@ -1,94 +0,0 @@
-#![feature(rustc_private)]
-#![warn(rust_2018_idioms)]
-#![warn(unused_lifetimes)]
-#![warn(unreachable_pub)]
-
-extern crate rustc_data_structures;
-extern crate rustc_driver;
-extern crate rustc_interface;
-extern crate rustc_session;
-extern crate rustc_target;
-
-use std::panic;
-
-use rustc_data_structures::profiling::{get_resident_set_size, print_time_passes_entry};
-use rustc_interface::interface;
-use rustc_session::config::{ErrorOutputType, TrimmedDefPaths};
-use rustc_session::early_error;
-use rustc_target::spec::PanicStrategy;
-
-// FIXME use std::lazy::SyncLazy once it stabilizes
-use once_cell::sync::Lazy;
-
-const BUG_REPORT_URL: &str = "https://github.com/bjorn3/rustc_codegen_cranelift/issues/new";
-
-static DEFAULT_HOOK: Lazy<Box<dyn Fn(&panic::PanicInfo<'_>) + Sync + Send + 'static>> =
-    Lazy::new(|| {
-        let hook = panic::take_hook();
-        panic::set_hook(Box::new(|info| {
-            // Invoke the default handler, which prints the actual panic message and optionally a backtrace
-            (*DEFAULT_HOOK)(info);
-
-            // Separate the output with an empty line
-            eprintln!();
-
-            // Print the ICE message
-            rustc_driver::report_ice(info, BUG_REPORT_URL);
-        }));
-        hook
-    });
-
-#[derive(Default)]
-pub struct CraneliftPassesCallbacks {
-    time_passes: bool,
-}
-
-impl rustc_driver::Callbacks for CraneliftPassesCallbacks {
-    fn config(&mut self, config: &mut interface::Config) {
-        // If a --prints=... option has been given, we don't print the "total"
-        // time because it will mess up the --prints output. See #64339.
-        self.time_passes = config.opts.prints.is_empty()
-            && (config.opts.debugging_opts.time_passes || config.opts.debugging_opts.time);
-
-        config.opts.cg.panic = Some(PanicStrategy::Abort);
-        config.opts.debugging_opts.panic_abort_tests = true;
-        config.opts.maybe_sysroot = Some(config.opts.maybe_sysroot.clone().unwrap_or_else(|| {
-            std::env::current_exe().unwrap().parent().unwrap().parent().unwrap().to_owned()
-        }));
-
-        config.opts.trimmed_def_paths = TrimmedDefPaths::GoodPath;
-    }
-}
-
-fn main() {
-    let start_time = std::time::Instant::now();
-    let start_rss = get_resident_set_size();
-    rustc_driver::init_rustc_env_logger();
-    let mut callbacks = CraneliftPassesCallbacks::default();
-    Lazy::force(&DEFAULT_HOOK); // Install ice hook
-    let exit_code = rustc_driver::catch_with_exit_code(|| {
-        let args = std::env::args_os()
-            .enumerate()
-            .map(|(i, arg)| {
-                arg.into_string().unwrap_or_else(|arg| {
-                    early_error(
-                        ErrorOutputType::default(),
-                        &format!("Argument {} is not valid Unicode: {:?}", i, arg),
-                    )
-                })
-            })
-            .collect::<Vec<_>>();
-        let mut run_compiler = rustc_driver::RunCompiler::new(&args, &mut callbacks);
-        run_compiler.set_make_codegen_backend(Some(Box::new(move |_| {
-            Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend { config: None })
-        })));
-        run_compiler.run()
-    });
-
-    if callbacks.time_passes {
-        let end_rss = get_resident_set_size();
-        print_time_passes_entry("total", start_time.elapsed(), start_rss, end_rss);
-    }
-
-    std::process::exit(exit_code)
-}
diff --git a/compiler/rustc_codegen_cranelift/src/bin/cg_clif_build_sysroot.rs b/compiler/rustc_codegen_cranelift/src/bin/cg_clif_build_sysroot.rs
deleted file mode 100644
index bde4d71b9a3..00000000000
--- a/compiler/rustc_codegen_cranelift/src/bin/cg_clif_build_sysroot.rs
+++ /dev/null
@@ -1,93 +0,0 @@
-//! The only difference between this and cg_clif.rs is that this binary defaults to using cg_llvm
-//! instead of cg_clif and requires `--clif` to use cg_clif and that this binary doesn't have JIT
-//! support.
-//! This is necessary as with Cargo `RUSTC` applies to both target crates and host crates. The host
-//! crates must be built with cg_llvm as we are currently building a sysroot for cg_clif.
-//! `RUSTFLAGS` however is only applied to target crates, so `--clif` would only be passed to the
-//! target crates.
-
-#![feature(rustc_private)]
-#![warn(rust_2018_idioms)]
-#![warn(unused_lifetimes)]
-#![warn(unreachable_pub)]
-
-extern crate rustc_driver;
-extern crate rustc_interface;
-extern crate rustc_session;
-extern crate rustc_target;
-
-use std::path::PathBuf;
-
-use rustc_interface::interface;
-use rustc_session::config::ErrorOutputType;
-use rustc_session::early_error;
-use rustc_target::spec::PanicStrategy;
-
-fn find_sysroot() -> String {
-    // Taken from https://github.com/Manishearth/rust-clippy/pull/911.
-    let home = option_env!("RUSTUP_HOME").or(option_env!("MULTIRUST_HOME"));
-    let toolchain = option_env!("RUSTUP_TOOLCHAIN").or(option_env!("MULTIRUST_TOOLCHAIN"));
-    match (home, toolchain) {
-        (Some(home), Some(toolchain)) => format!("{}/toolchains/{}", home, toolchain),
-        _ => option_env!("RUST_SYSROOT")
-            .expect("need to specify RUST_SYSROOT env var or use rustup or multirust")
-            .to_owned(),
-    }
-}
-
-pub struct CraneliftPassesCallbacks {
-    use_clif: bool,
-}
-
-impl rustc_driver::Callbacks for CraneliftPassesCallbacks {
-    fn config(&mut self, config: &mut interface::Config) {
-        if !self.use_clif {
-            config.opts.maybe_sysroot = Some(PathBuf::from(find_sysroot()));
-            return;
-        }
-
-        config.opts.cg.panic = Some(PanicStrategy::Abort);
-        config.opts.debugging_opts.panic_abort_tests = true;
-        config.opts.maybe_sysroot =
-            Some(std::env::current_exe().unwrap().parent().unwrap().parent().unwrap().to_owned());
-    }
-}
-
-fn main() {
-    rustc_driver::init_rustc_env_logger();
-    rustc_driver::install_ice_hook();
-    let exit_code = rustc_driver::catch_with_exit_code(|| {
-        let mut use_clif = false;
-
-        let args = std::env::args_os()
-            .enumerate()
-            .map(|(i, arg)| {
-                arg.into_string().unwrap_or_else(|arg| {
-                    early_error(
-                        ErrorOutputType::default(),
-                        &format!("Argument {} is not valid Unicode: {:?}", i, arg),
-                    )
-                })
-            })
-            .filter(|arg| {
-                if arg == "--clif" {
-                    use_clif = true;
-                    false
-                } else {
-                    true
-                }
-            })
-            .collect::<Vec<_>>();
-
-        let mut callbacks = CraneliftPassesCallbacks { use_clif };
-
-        let mut run_compiler = rustc_driver::RunCompiler::new(&args, &mut callbacks);
-        if use_clif {
-            run_compiler.set_make_codegen_backend(Some(Box::new(move |_| {
-                Box::new(rustc_codegen_cranelift::CraneliftCodegenBackend { config: None })
-            })));
-        }
-        run_compiler.run()
-    });
-    std::process::exit(exit_code)
-}
diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs
index ffa629ca16c..f9dc1b5169e 100644
--- a/compiler/rustc_codegen_cranelift/src/common.rs
+++ b/compiler/rustc_codegen_cranelift/src/common.rs
@@ -340,22 +340,46 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         self.bcx.set_srcloc(SourceLoc::new(index as u32));
     }
 
-    pub(crate) fn get_caller_location(&mut self, span: Span) -> CValue<'tcx> {
-        if let Some(loc) = self.caller_location {
-            // `#[track_caller]` is used; return caller location instead of current location.
-            return loc;
+    // Note: must be kept in sync with get_caller_location from cg_ssa
+    pub(crate) fn get_caller_location(&mut self, mut source_info: mir::SourceInfo) -> CValue<'tcx> {
+        let span_to_caller_location = |fx: &mut FunctionCx<'_, '_, 'tcx>, span: Span| {
+            let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span);
+            let caller = fx.tcx.sess.source_map().lookup_char_pos(topmost.lo());
+            let const_loc = fx.tcx.const_caller_location((
+                rustc_span::symbol::Symbol::intern(
+                    &caller.file.name.prefer_remapped().to_string_lossy(),
+                ),
+                caller.line as u32,
+                caller.col_display as u32 + 1,
+            ));
+            crate::constant::codegen_const_value(fx, const_loc, fx.tcx.caller_location_ty())
+        };
+
+        // Walk up the `SourceScope`s, in case some of them are from MIR inlining.
+        // If so, the starting `source_info.span` is in the innermost inlined
+        // function, and will be replaced with outer callsite spans as long
+        // as the inlined functions were `#[track_caller]`.
+        loop {
+            let scope_data = &self.mir.source_scopes[source_info.scope];
+
+            if let Some((callee, callsite_span)) = scope_data.inlined {
+                // Stop inside the most nested non-`#[track_caller]` function,
+                // before ever reaching its caller (which is irrelevant).
+                if !callee.def.requires_caller_location(self.tcx) {
+                    return span_to_caller_location(self, source_info.span);
+                }
+                source_info.span = callsite_span;
+            }
+
+            // Skip past all of the parents with `inlined: None`.
+            match scope_data.inlined_parent_scope {
+                Some(parent) => source_info.scope = parent,
+                None => break,
+            }
         }
 
-        let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span);
-        let caller = self.tcx.sess.source_map().lookup_char_pos(topmost.lo());
-        let const_loc = self.tcx.const_caller_location((
-            rustc_span::symbol::Symbol::intern(
-                &caller.file.name.prefer_remapped().to_string_lossy(),
-            ),
-            caller.line as u32,
-            caller.col_display as u32 + 1,
-        ));
-        crate::constant::codegen_const_value(self, const_loc, self.tcx.caller_location_ty())
+        // No inlined `SourceScope`s, or all of them were `#[track_caller]`.
+        self.caller_location.unwrap_or_else(|| span_to_caller_location(self, source_info.span))
     }
 
     pub(crate) fn anonymous_str(&mut self, msg: &str) -> Value {
diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
index 7f15bc75fda..1b01f4edbb3 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
@@ -74,6 +74,7 @@ fn create_jit_module<'tcx>(
     jit_builder.hotswap(hotswap);
     crate::compiler_builtins::register_functions_for_jit(&mut jit_builder);
     jit_builder.symbols(imported_symbols);
+    jit_builder.symbol("__clif_jit_fn", clif_jit_fn as *const u8);
     let mut jit_module = JITModule::new(jit_builder);
 
     let mut cx = crate::CodegenCx::new(
@@ -210,8 +211,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
     }
 }
 
-#[no_mangle]
-extern "C" fn __clif_jit_fn(
+extern "C" fn clif_jit_fn(
     instance_ptr: *const Instance<'static>,
     trampoline_ptr: *const u8,
 ) -> *const u8 {
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
index f7a83373e87..29b3f36b2be 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
@@ -218,7 +218,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
     instance: Instance<'tcx>,
     args: &[mir::Operand<'tcx>],
     destination: Option<(CPlace<'tcx>, BasicBlock)>,
-    span: Span,
+    source_info: mir::SourceInfo,
 ) {
     let intrinsic = fx.tcx.item_name(instance.def_id());
     let substs = instance.substs;
@@ -232,7 +232,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
                     fx.bcx.ins().trap(TrapCode::User(0));
                 }
                 sym::transmute => {
-                    crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", span);
+                    crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", source_info);
                 }
                 _ => unimplemented!("unsupported instrinsic {}", intrinsic),
             }
@@ -241,7 +241,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
     };
 
     if intrinsic.as_str().starts_with("simd_") {
-        self::simd::codegen_simd_intrinsic_call(fx, intrinsic, substs, args, ret, span);
+        self::simd::codegen_simd_intrinsic_call(fx, intrinsic, substs, args, ret, source_info.span);
         let ret_block = fx.get_block(destination.expect("SIMD intrinsics don't diverge").1);
         fx.bcx.ins().jump(ret_block, &[]);
     } else if codegen_float_intrinsic_call(fx, intrinsic, args, ret) {
@@ -255,7 +255,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             substs,
             args,
             ret,
-            span,
+            source_info,
             destination,
         );
     }
@@ -339,7 +339,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
     substs: SubstsRef<'tcx>,
     args: &[mir::Operand<'tcx>],
     ret: CPlace<'tcx>,
-    span: Span,
+    source_info: mir::SourceInfo,
     destination: Option<(CPlace<'tcx>, BasicBlock)>,
 ) {
     let usize_layout = fx.layout_of(fx.tcx.types.usize);
@@ -347,7 +347,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
     intrinsic_match! {
         fx, intrinsic, args,
         _ => {
-            fx.tcx.sess.span_fatal(span, &format!("unsupported intrinsic {}", intrinsic));
+            fx.tcx.sess.span_fatal(source_info.span, &format!("unsupported intrinsic {}", intrinsic));
         };
 
         assume, (c _a) {};
@@ -658,7 +658,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
                     crate::base::codegen_panic(
                         fx,
                         &format!("attempted to instantiate uninhabited type `{}`", layout.ty),
-                        span,
+                        source_info,
                     )
                 });
                 return;
@@ -669,7 +669,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
                     crate::base::codegen_panic(
                         fx,
                         &format!("attempted to zero-initialize type `{}`, which is invalid", layout.ty),
-                        span,
+                        source_info,
                     );
                 });
                 return;
@@ -680,7 +680,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
                     crate::base::codegen_panic(
                         fx,
                         &format!("attempted to leave type `{}` uninitialized, which is invalid", layout.ty),
-                        span,
+                        source_info,
                     )
                 });
                 return;
@@ -715,19 +715,19 @@ fn codegen_regular_intrinsic_call<'tcx>(
 
         ptr_offset_from | ptr_offset_from_unsigned, (v ptr, v base) {
             let ty = substs.type_at(0);
-            let isize_layout = fx.layout_of(fx.tcx.types.isize);
 
             let pointee_size: u64 = fx.layout_of(ty).size.bytes();
             let diff_bytes = fx.bcx.ins().isub(ptr, base);
             // FIXME this can be an exact division.
-            let diff = if intrinsic == sym::ptr_offset_from_unsigned {
+            let val = if intrinsic == sym::ptr_offset_from_unsigned {
+                let usize_layout = fx.layout_of(fx.tcx.types.usize);
                 // Because diff_bytes ULE isize::MAX, this would be fine as signed,
                 // but unsigned is slightly easier to codegen, so might as well.
-                fx.bcx.ins().udiv_imm(diff_bytes, pointee_size as i64)
+                CValue::by_val(fx.bcx.ins().udiv_imm(diff_bytes, pointee_size as i64), usize_layout)
             } else {
-                fx.bcx.ins().sdiv_imm(diff_bytes, pointee_size as i64)
+                let isize_layout = fx.layout_of(fx.tcx.types.isize);
+                CValue::by_val(fx.bcx.ins().sdiv_imm(diff_bytes, pointee_size as i64), isize_layout)
             };
-            let val = CValue::by_val(diff, isize_layout);
             ret.write_cvalue(fx, val);
         };
 
@@ -742,7 +742,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
         };
 
         caller_location, () {
-            let caller_location = fx.get_caller_location(span);
+            let caller_location = fx.get_caller_location(source_info);
             ret.write_cvalue(fx, caller_location);
         };
 
@@ -765,12 +765,12 @@ fn codegen_regular_intrinsic_call<'tcx>(
                         fx.bcx.ins().jump(ret_block, &[]);
                         return;
                     } else {
-                        fx.tcx.sess.span_fatal(span, "128bit atomics not yet supported");
+                        fx.tcx.sess.span_fatal(source_info.span, "128bit atomics not yet supported");
                     }
                 }
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
-                    report_atomic_type_validation_error(fx, intrinsic, span, ty);
+                    report_atomic_type_validation_error(fx, intrinsic, source_info.span, ty);
                     return;
                 }
             }
@@ -793,12 +793,12 @@ fn codegen_regular_intrinsic_call<'tcx>(
                         fx.bcx.ins().jump(ret_block, &[]);
                         return;
                     } else {
-                        fx.tcx.sess.span_fatal(span, "128bit atomics not yet supported");
+                        fx.tcx.sess.span_fatal(source_info.span, "128bit atomics not yet supported");
                     }
                 }
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
-                    report_atomic_type_validation_error(fx, intrinsic, span, ty);
+                    report_atomic_type_validation_error(fx, intrinsic, source_info.span, ty);
                     return;
                 }
             }
@@ -812,7 +812,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             match layout.ty.kind() {
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
-                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
                     return;
                 }
             }
@@ -830,7 +830,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             match layout.ty.kind() {
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
-                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
                     return;
                 }
             }
@@ -850,7 +850,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             match layout.ty.kind() {
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
-                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
                     return;
                 }
             }
@@ -868,7 +868,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             match layout.ty.kind() {
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
-                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
                     return;
                 }
             }
@@ -886,7 +886,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             match layout.ty.kind() {
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
-                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
                     return;
                 }
             }
@@ -904,7 +904,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             match layout.ty.kind() {
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
-                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
                     return;
                 }
             }
@@ -922,7 +922,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             match layout.ty.kind() {
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
-                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
                     return;
                 }
             }
@@ -940,7 +940,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             match layout.ty.kind() {
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
-                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
                     return;
                 }
             }
@@ -958,7 +958,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             match layout.ty.kind() {
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
-                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
                     return;
                 }
             }
@@ -976,7 +976,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             match layout.ty.kind() {
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
-                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
                     return;
                 }
             }
@@ -994,7 +994,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             match layout.ty.kind() {
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
-                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
                     return;
                 }
             }
@@ -1012,7 +1012,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             match layout.ty.kind() {
                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
                 _ => {
-                    report_atomic_type_validation_error(fx, intrinsic, span, layout.ty);
+                    report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
                     return;
                 }
             }
diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
index 8f80b02ae0d..a68225de58b 100644
--- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs
+++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
@@ -828,6 +828,7 @@ pub(crate) fn assert_assignable<'tcx>(
                 }
             }
         }
+        (ty::Array(a, _), ty::Array(b, _)) => assert_assignable(fx, *a, *b),
         _ => {
             assert_eq!(
                 from_ty, to_ty,