Merge commit '3a31c6d8272c14388a34622193baf553636fe470' into sync_cg_clif-2021-07-07

13 files changed, 306 insertions, 126 deletions

diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs
index ec3e17e5b75..3d78eed77b9 100644
--- a/compiler/rustc_codegen_cranelift/src/base.rs
+++ b/compiler/rustc_codegen_cranelift/src/base.rs
@@ -21,6 +21,11 @@ pub(crate) fn codegen_fn<'tcx>(
     debug_assert!(!instance.substs.needs_infer());
 
     let mir = tcx.instance_mir(instance.def);
+    let _mir_guard = crate::PrintOnPanic(|| {
+        let mut buf = Vec::new();
+        rustc_mir::util::write_mir_pretty(tcx, Some(instance.def_id()), &mut buf).unwrap();
+        String::from_utf8_lossy(&buf).into_owned()
+    });
 
     // Declare function
     let symbol_name = tcx.symbol_name(instance);
@@ -52,7 +57,6 @@ pub(crate) fn codegen_fn<'tcx>(
         module,
         tcx,
         pointer_type,
-        vtables: FxHashMap::default(),
         constants_cx: ConstantCx::new(),
 
         instance,
@@ -105,7 +109,14 @@ pub(crate) fn codegen_fn<'tcx>(
     let context = &mut cx.cached_context;
     context.func = func;
 
-    crate::pretty_clif::write_clif_file(tcx, "unopt", None, instance, &context, &clif_comments);
+    crate::pretty_clif::write_clif_file(
+        tcx,
+        "unopt",
+        module.isa(),
+        instance,
+        &context,
+        &clif_comments,
+    );
 
     // Verify function
     verify_func(tcx, &clif_comments, &context.func);
@@ -122,7 +133,13 @@ pub(crate) fn codegen_fn<'tcx>(
 
     // Perform rust specific optimizations
     tcx.sess.time("optimize clif ir", || {
-        crate::optimize::optimize_function(tcx, instance, context, &mut clif_comments);
+        crate::optimize::optimize_function(
+            tcx,
+            module.isa(),
+            instance,
+            context,
+            &mut clif_comments,
+        );
     });
 
     // Define function
@@ -137,7 +154,7 @@ pub(crate) fn codegen_fn<'tcx>(
     crate::pretty_clif::write_clif_file(
         tcx,
         "opt",
-        Some(module.isa()),
+        module.isa(),
         instance,
         &context,
         &clif_comments,
diff --git a/compiler/rustc_codegen_cranelift/src/bin/cg_clif.rs b/compiler/rustc_codegen_cranelift/src/bin/cg_clif.rs
index 2643fae0a81..a044b43b864 100644
--- a/compiler/rustc_codegen_cranelift/src/bin/cg_clif.rs
+++ b/compiler/rustc_codegen_cranelift/src/bin/cg_clif.rs
@@ -6,8 +6,8 @@ extern crate rustc_interface;
 extern crate rustc_session;
 extern crate rustc_target;
 
-use std::panic;
 use std::lazy::SyncLazy;
+use std::panic;
 
 use rustc_data_structures::profiling::{get_resident_set_size, print_time_passes_entry};
 use rustc_interface::interface;
diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs
index a8a0bb52a24..892ccf27f6d 100644
--- a/compiler/rustc_codegen_cranelift/src/common.rs
+++ b/compiler/rustc_codegen_cranelift/src/common.rs
@@ -233,7 +233,6 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
     pub(crate) module: &'m mut dyn Module,
     pub(crate) tcx: TyCtxt<'tcx>,
     pub(crate) pointer_type: Type, // Cached from module
-    pub(crate) vtables: FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), Pointer>,
     pub(crate) constants_cx: ConstantCx,
 
     pub(crate) instance: Instance<'tcx>,
diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs
index a87b3703949..2a2573aad29 100644
--- a/compiler/rustc_codegen_cranelift/src/constant.rs
+++ b/compiler/rustc_codegen_cranelift/src/constant.rs
@@ -1,16 +1,13 @@
 //! Handling of `static`s, `const`s and promoted allocations
 
-use rustc_span::DUMMY_SP;
-
-use rustc_ast::Mutability;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_errors::ErrorReported;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::interpret::{
-    alloc_range, read_target_uint, AllocId, Allocation, ConstValue, ErrorHandled, GlobalAlloc,
-    Scalar,
+    read_target_uint, AllocId, Allocation, ConstValue, ErrorHandled, GlobalAlloc, Scalar,
 };
 use rustc_middle::ty::ConstKind;
+use rustc_span::DUMMY_SP;
 
 use cranelift_codegen::ir::GlobalValueData;
 use cranelift_module::*;
@@ -171,66 +168,74 @@ pub(crate) fn codegen_const_value<'tcx>(
     }
 
     match const_val {
-        ConstValue::Scalar(x) => {
-            if fx.clif_type(layout.ty).is_none() {
-                let (size, align) = (layout.size, layout.align.pref);
-                let mut alloc = Allocation::from_bytes(
-                    std::iter::repeat(0).take(size.bytes_usize()).collect::<Vec<u8>>(),
-                    align,
-                    Mutability::Not,
-                );
-                alloc.write_scalar(fx, alloc_range(Size::ZERO, size), x.into()).unwrap();
-                let alloc = fx.tcx.intern_const_alloc(alloc);
-                return CValue::by_ref(pointer_for_allocation(fx, alloc), layout);
-            }
-
-            match x {
-                Scalar::Int(int) => CValue::const_val(fx, layout, int),
-                Scalar::Ptr(ptr) => {
-                    let alloc_kind = fx.tcx.get_global_alloc(ptr.alloc_id);
-                    let base_addr = match alloc_kind {
-                        Some(GlobalAlloc::Memory(alloc)) => {
-                            fx.constants_cx.todo.push(TodoItem::Alloc(ptr.alloc_id));
-                            let data_id = data_id_for_alloc_id(
-                                &mut fx.constants_cx,
-                                fx.module,
-                                ptr.alloc_id,
-                                alloc.mutability,
-                            );
-                            let local_data_id =
-                                fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-                            if fx.clif_comments.enabled() {
-                                fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id));
-                            }
-                            fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
-                        }
-                        Some(GlobalAlloc::Function(instance)) => {
-                            let func_id = crate::abi::import_function(fx.tcx, fx.module, instance);
-                            let local_func_id =
-                                fx.module.declare_func_in_func(func_id, &mut fx.bcx.func);
-                            fx.bcx.ins().func_addr(fx.pointer_type, local_func_id)
-                        }
-                        Some(GlobalAlloc::Static(def_id)) => {
-                            assert!(fx.tcx.is_static(def_id));
-                            let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false);
-                            let local_data_id =
-                                fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-                            if fx.clif_comments.enabled() {
-                                fx.add_comment(local_data_id, format!("{:?}", def_id));
-                            }
-                            fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
+        ConstValue::Scalar(x) => match x {
+            Scalar::Int(int) => {
+                if fx.clif_type(layout.ty).is_some() {
+                    return CValue::const_val(fx, layout, int);
+                } else {
+                    let raw_val = int.to_bits(int.size()).unwrap();
+                    let val = match int.size().bytes() {
+                        1 => fx.bcx.ins().iconst(types::I8, raw_val as i64),
+                        2 => fx.bcx.ins().iconst(types::I16, raw_val as i64),
+                        4 => fx.bcx.ins().iconst(types::I32, raw_val as i64),
+                        8 => fx.bcx.ins().iconst(types::I64, raw_val as i64),
+                        16 => {
+                            let lsb = fx.bcx.ins().iconst(types::I64, raw_val as u64 as i64);
+                            let msb =
+                                fx.bcx.ins().iconst(types::I64, (raw_val >> 64) as u64 as i64);
+                            fx.bcx.ins().iconcat(lsb, msb)
                         }
-                        None => bug!("missing allocation {:?}", ptr.alloc_id),
-                    };
-                    let val = if ptr.offset.bytes() != 0 {
-                        fx.bcx.ins().iadd_imm(base_addr, i64::try_from(ptr.offset.bytes()).unwrap())
-                    } else {
-                        base_addr
+                        _ => unreachable!(),
                     };
-                    CValue::by_val(val, layout)
+
+                    let place = CPlace::new_stack_slot(fx, layout);
+                    place.to_ptr().store(fx, val, MemFlags::trusted());
+                    place.to_cvalue(fx)
                 }
             }
-        }
+            Scalar::Ptr(ptr) => {
+                let alloc_kind = fx.tcx.get_global_alloc(ptr.alloc_id);
+                let base_addr = match alloc_kind {
+                    Some(GlobalAlloc::Memory(alloc)) => {
+                        let data_id = data_id_for_alloc_id(
+                            &mut fx.constants_cx,
+                            fx.module,
+                            ptr.alloc_id,
+                            alloc.mutability,
+                        );
+                        let local_data_id =
+                            fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
+                        if fx.clif_comments.enabled() {
+                            fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id));
+                        }
+                        fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
+                    }
+                    Some(GlobalAlloc::Function(instance)) => {
+                        let func_id = crate::abi::import_function(fx.tcx, fx.module, instance);
+                        let local_func_id =
+                            fx.module.declare_func_in_func(func_id, &mut fx.bcx.func);
+                        fx.bcx.ins().func_addr(fx.pointer_type, local_func_id)
+                    }
+                    Some(GlobalAlloc::Static(def_id)) => {
+                        assert!(fx.tcx.is_static(def_id));
+                        let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false);
+                        let local_data_id =
+                            fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
+                        if fx.clif_comments.enabled() {
+                            fx.add_comment(local_data_id, format!("{:?}", def_id));
+                        }
+                        fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
+                    }
+                    None => bug!("missing allocation {:?}", ptr.alloc_id),
+                };
+                let val = if ptr.offset.bytes() != 0 {
+                    fx.bcx.ins().iadd_imm(base_addr, i64::try_from(ptr.offset.bytes()).unwrap())
+                } else {
+                    base_addr
+                };
+                CValue::by_val(val, layout)
+            }
+        },
         ConstValue::ByRef { alloc, offset } => CValue::by_ref(
             pointer_for_allocation(fx, alloc)
                 .offset_i64(fx, i64::try_from(offset.bytes()).unwrap()),
@@ -254,7 +259,6 @@ pub(crate) fn pointer_for_allocation<'tcx>(
     alloc: &'tcx Allocation,
 ) -> crate::pointer::Pointer {
     let alloc_id = fx.tcx.create_memory_alloc(alloc);
-    fx.constants_cx.todo.push(TodoItem::Alloc(alloc_id));
     let data_id =
         data_id_for_alloc_id(&mut fx.constants_cx, &mut *fx.module, alloc_id, alloc.mutability);
 
@@ -266,12 +270,13 @@ pub(crate) fn pointer_for_allocation<'tcx>(
     crate::pointer::Pointer::new(global_ptr)
 }
 
-fn data_id_for_alloc_id(
+pub(crate) fn data_id_for_alloc_id(
     cx: &mut ConstantCx,
     module: &mut dyn Module,
     alloc_id: AllocId,
     mutability: rustc_hir::Mutability,
 ) -> DataId {
+    cx.todo.push(TodoItem::Alloc(alloc_id));
     *cx.anon_allocs.entry(alloc_id).or_insert_with(|| {
         module.declare_anonymous_data(mutability == rustc_hir::Mutability::Mut, false).unwrap()
     })
@@ -352,7 +357,14 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
                     GlobalAlloc::Memory(alloc) => alloc,
                     GlobalAlloc::Function(_) | GlobalAlloc::Static(_) => unreachable!(),
                 };
-                let data_id = data_id_for_alloc_id(cx, module, alloc_id, alloc.mutability);
+                let data_id = *cx.anon_allocs.entry(alloc_id).or_insert_with(|| {
+                    module
+                        .declare_anonymous_data(
+                            alloc.mutability == rustc_hir::Mutability::Mut,
+                            false,
+                        )
+                        .unwrap()
+                });
                 (data_id, alloc, None)
             }
             TodoItem::Static(def_id) => {
@@ -415,7 +427,6 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
                     continue;
                 }
                 GlobalAlloc::Memory(target_alloc) => {
-                    cx.todo.push(TodoItem::Alloc(reloc));
                     data_id_for_alloc_id(cx, module, reloc, target_alloc.mutability)
                 }
                 GlobalAlloc::Static(def_id) => {
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
index 9eb06770630..c7e15f81e03 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
@@ -10,7 +10,7 @@ use rustc_span::{
 };
 
 use cranelift_codegen::binemit::CodeOffset;
-use cranelift_codegen::machinst::MachSrcLoc;
+use cranelift_codegen::MachSrcLoc;
 
 use gimli::write::{
     Address, AttributeValue, FileId, FileInfo, LineProgram, LineString, LineStringTable,
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
index 61e54a76f29..c67336eb3f2 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
@@ -61,9 +61,11 @@ impl<'tcx> DebugContext<'tcx> {
 
         let mut dwarf = DwarfUnit::new(encoding);
 
-        // FIXME: how to get version when building out of tree?
-        // Normally this would use option_env!("CFG_VERSION").
-        let producer = format!("cg_clif (rustc {})", "unknown version");
+        let producer = format!(
+            "cg_clif (rustc {}, cranelift {})",
+            rustc_interface::util::version_str().unwrap_or("unknown version"),
+            cranelift_codegen::VERSION,
+        );
         let comp_dir = tcx.sess.working_dir.to_string_lossy(false).into_owned();
         let (name, file_info) = match tcx.sess.local_crate_source_file.clone() {
             Some(path) => {
diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs
index 50fd53481f7..a8b802f4494 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs
@@ -289,13 +289,16 @@ pub(crate) fn run_aot(
         None
     };
 
+    // FIXME handle `-Ctarget-cpu=native`
+    let target_cpu =
+        tcx.sess.opts.cg.target_cpu.as_ref().unwrap_or(&tcx.sess.target.cpu).to_owned();
     Box::new((
         CodegenResults {
             modules,
             allocator_module,
             metadata_module,
             metadata,
-            crate_info: CrateInfo::new(tcx, crate::target_triple(tcx.sess).to_string()),
+            crate_info: CrateInfo::new(tcx, target_cpu),
         },
         work_products,
     ))
diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
index 04ec01ad281..76fbc9ad51e 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
@@ -3,11 +3,14 @@
 
 use std::cell::RefCell;
 use std::ffi::CString;
+use std::lazy::{Lazy, SyncOnceCell};
 use std::os::raw::{c_char, c_int};
+use std::sync::{mpsc, Mutex};
 
 use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_codegen_ssa::CrateInfo;
 use rustc_middle::mir::mono::MonoItem;
+use rustc_session::Session;
 
 use cranelift_jit::{JITBuilder, JITModule};
 
@@ -23,12 +26,48 @@ thread_local! {
     static LAZY_JIT_STATE: RefCell<Option<JitState>> = RefCell::new(None);
 }
 
+/// The Sender owned by the rustc thread
+static GLOBAL_MESSAGE_SENDER: SyncOnceCell<Mutex<mpsc::Sender<UnsafeMessage>>> =
+    SyncOnceCell::new();
+
+/// A message that is sent from the jitted runtime to the rustc thread.
+/// Senders are responsible for upholding `Send` semantics.
+enum UnsafeMessage {
+    /// Request that the specified `Instance` be lazily jitted.
+    ///
+    /// Nothing accessible through `instance_ptr` may be moved or mutated by the sender after
+    /// this message is sent.
+    JitFn {
+        instance_ptr: *const Instance<'static>,
+        trampoline_ptr: *const u8,
+        tx: mpsc::Sender<*const u8>,
+    },
+}
+unsafe impl Send for UnsafeMessage {}
+
+impl UnsafeMessage {
+    /// Send the message.
+    fn send(self) -> Result<(), mpsc::SendError<UnsafeMessage>> {
+        thread_local! {
+            /// The Sender owned by the local thread
+            static LOCAL_MESSAGE_SENDER: Lazy<mpsc::Sender<UnsafeMessage>> = Lazy::new(||
+                GLOBAL_MESSAGE_SENDER
+                    .get().unwrap()
+                    .lock().unwrap()
+                    .clone()
+            );
+        }
+        LOCAL_MESSAGE_SENDER.with(|sender| sender.send(self))
+    }
+}
+
 fn create_jit_module<'tcx>(
     tcx: TyCtxt<'tcx>,
     backend_config: &BackendConfig,
     hotswap: bool,
 ) -> (JITModule, CodegenCx<'tcx>) {
-    let imported_symbols = load_imported_symbols_for_jit(tcx);
+    let crate_info = CrateInfo::new(tcx, "dummy_target_cpu".to_string());
+    let imported_symbols = load_imported_symbols_for_jit(tcx.sess, crate_info);
 
     let isa = crate::build_isa(tcx.sess, backend_config);
     let mut jit_builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names());
@@ -116,11 +155,6 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
         .chain(backend_config.jit_args.iter().map(|arg| &**arg))
         .map(|arg| CString::new(arg).unwrap())
         .collect::<Vec<_>>();
-    let mut argv = args.iter().map(|arg| arg.as_ptr()).collect::<Vec<_>>();
-
-    // Push a null pointer as a terminating argument. This is required by POSIX and
-    // useful as some dynamic linkers use it as a marker to jump over.
-    argv.push(std::ptr::null());
 
     let start_sig = Signature {
         params: vec![
@@ -128,7 +162,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
             AbiParam::new(jit_module.target_config().pointer_type()),
         ],
         returns: vec![AbiParam::new(jit_module.target_config().pointer_type() /*isize*/)],
-        call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
+        call_conv: jit_module.target_config().default_call_conv,
     };
     let start_func_id = jit_module.declare_function("main", Linkage::Import, &start_sig).unwrap();
     let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id);
@@ -141,12 +175,51 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
 
     let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
         unsafe { ::std::mem::transmute(finalized_start) };
-    let ret = f(args.len() as c_int, argv.as_ptr());
-    std::process::exit(ret);
+
+    let (tx, rx) = mpsc::channel();
+    GLOBAL_MESSAGE_SENDER.set(Mutex::new(tx)).unwrap();
+
+    // Spawn the jitted runtime in a new thread so that this rustc thread can handle messages
+    // (eg to lazily JIT further functions as required)
+    std::thread::spawn(move || {
+        let mut argv = args.iter().map(|arg| arg.as_ptr()).collect::<Vec<_>>();
+
+        // Push a null pointer as a terminating argument. This is required by POSIX and
+        // useful as some dynamic linkers use it as a marker to jump over.
+        argv.push(std::ptr::null());
+
+        let ret = f(args.len() as c_int, argv.as_ptr());
+        std::process::exit(ret);
+    });
+
+    // Handle messages
+    loop {
+        match rx.recv().unwrap() {
+            // lazy JIT compilation request - compile requested instance and return pointer to result
+            UnsafeMessage::JitFn { instance_ptr, trampoline_ptr, tx } => {
+                tx.send(jit_fn(instance_ptr, trampoline_ptr))
+                    .expect("jitted runtime hung up before response to lazy JIT request was sent");
+            }
+        }
+    }
 }
 
 #[no_mangle]
-extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 {
+extern "C" fn __clif_jit_fn(
+    instance_ptr: *const Instance<'static>,
+    trampoline_ptr: *const u8,
+) -> *const u8 {
+    // send the JIT request to the rustc thread, with a channel for the response
+    let (tx, rx) = mpsc::channel();
+    UnsafeMessage::JitFn { instance_ptr, trampoline_ptr, tx }
+        .send()
+        .expect("rustc thread hung up before lazy JIT request was sent");
+
+    // block on JIT compilation result
+    rx.recv().expect("rustc thread hung up before responding to sent lazy JIT request")
+}
+
+fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) -> *const u8 {
     rustc_middle::ty::tls::with(|tcx| {
         // lift is used to ensure the correct lifetime for instance.
         let instance = tcx.lift(unsafe { *instance_ptr }).unwrap();
@@ -160,6 +233,17 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8
             let name = tcx.symbol_name(instance).name;
             let sig = crate::abi::get_function_sig(tcx, jit_module.isa().triple(), instance);
             let func_id = jit_module.declare_function(name, Linkage::Export, &sig).unwrap();
+
+            let current_ptr = jit_module.read_got_entry(func_id);
+
+            // If the function's GOT entry has already been updated to point at something other
+            // than the shim trampoline, don't re-jit but just return the new pointer instead.
+            // This does not need synchronization as this code is executed only by a sole rustc
+            // thread.
+            if current_ptr != trampoline_ptr {
+                return current_ptr;
+            }
+
             jit_module.prepare_for_function_redefine(func_id).unwrap();
 
             let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module.isa(), false);
@@ -173,14 +257,16 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8
     })
 }
 
-fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
+fn load_imported_symbols_for_jit(
+    sess: &Session,
+    crate_info: CrateInfo,
+) -> Vec<(String, *const u8)> {
     use rustc_middle::middle::dependency_format::Linkage;
 
     let mut dylib_paths = Vec::new();
 
-    let crate_info = CrateInfo::new(tcx, "dummy_target_cpu".to_string());
-    let formats = tcx.dependency_formats(());
-    let data = &formats
+    let data = &crate_info
+        .dependency_formats
         .iter()
         .find(|(crate_type, _data)| *crate_type == rustc_session::config::CrateType::Executable)
         .unwrap()
@@ -190,9 +276,8 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
         match data[cnum.as_usize() - 1] {
             Linkage::NotLinked | Linkage::IncludedFromDylib => {}
             Linkage::Static => {
-                let name = tcx.crate_name(cnum);
-                let mut err =
-                    tcx.sess.struct_err(&format!("Can't load static lib {}", name.as_str()));
+                let name = &crate_info.crate_name[&cnum];
+                let mut err = sess.struct_err(&format!("Can't load static lib {}", name.as_str()));
                 err.note("rustc_codegen_cranelift can only load dylibs in JIT mode.");
                 err.emit();
             }
@@ -232,7 +317,7 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
         std::mem::forget(lib)
     }
 
-    tcx.sess.abort_if_errors();
+    sess.abort_if_errors();
 
     imported_symbols
 }
@@ -254,7 +339,7 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: In
             Linkage::Import,
             &Signature {
                 call_conv: module.target_config().default_call_conv,
-                params: vec![AbiParam::new(pointer_type)],
+                params: vec![AbiParam::new(pointer_type), AbiParam::new(pointer_type)],
                 returns: vec![AbiParam::new(pointer_type)],
             },
         )
@@ -267,6 +352,7 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: In
     let mut builder_ctx = FunctionBuilderContext::new();
     let mut trampoline_builder = FunctionBuilder::new(trampoline, &mut builder_ctx);
 
+    let trampoline_fn = module.declare_func_in_func(func_id, trampoline_builder.func);
     let jit_fn = module.declare_func_in_func(jit_fn, trampoline_builder.func);
     let sig_ref = trampoline_builder.func.import_signature(sig);
 
@@ -276,7 +362,8 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: In
 
     trampoline_builder.switch_to_block(entry_block);
     let instance_ptr = trampoline_builder.ins().iconst(pointer_type, instance_ptr as u64 as i64);
-    let jitted_fn = trampoline_builder.ins().call(jit_fn, &[instance_ptr]);
+    let trampoline_ptr = trampoline_builder.ins().func_addr(pointer_type, trampoline_fn);
+    let jitted_fn = trampoline_builder.ins().call(jit_fn, &[instance_ptr, trampoline_ptr]);
     let jitted_fn = trampoline_builder.func.dfg.inst_results(jitted_fn)[0];
     let call_inst = trampoline_builder.ins().call_indirect(sig_ref, jitted_fn, &fn_args);
     let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec();
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
index ba4ed2162cd..be3704ca276 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
@@ -106,6 +106,26 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
             let dest = CPlace::for_ptr(Pointer::new(mem_addr), a.layout());
             dest.write_cvalue(fx, a);
         };
+        "llvm.x86.addcarry.64", (v c_in, c a, c b) {
+            llvm_add_sub(
+                fx,
+                BinOp::Add,
+                ret,
+                c_in,
+                a,
+                b
+            );
+        };
+        "llvm.x86.subborrow.64", (v b_in, c a, c b) {
+            llvm_add_sub(
+                fx,
+                BinOp::Sub,
+                ret,
+                b_in,
+                a,
+                b
+            );
+        };
     }
 
     if let Some((_, dest)) = destination {
@@ -121,3 +141,41 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
 // llvm.x86.avx2.pshuf.b
 // llvm.x86.avx2.psrli.w
 // llvm.x86.sse2.psrli.w
+
+fn llvm_add_sub<'tcx>(
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
+    bin_op: BinOp,
+    ret: CPlace<'tcx>,
+    cb_in: Value,
+    a: CValue<'tcx>,
+    b: CValue<'tcx>,
+) {
+    assert_eq!(
+        a.layout().ty,
+        fx.tcx.types.u64,
+        "llvm.x86.addcarry.64/llvm.x86.subborrow.64 second operand must be u64"
+    );
+    assert_eq!(
+        b.layout().ty,
+        fx.tcx.types.u64,
+        "llvm.x86.addcarry.64/llvm.x86.subborrow.64 third operand must be u64"
+    );
+
+    // c + carry -> c + first intermediate carry or borrow respectively
+    let int0 = crate::num::codegen_checked_int_binop(fx, bin_op, a, b);
+    let c = int0.value_field(fx, mir::Field::new(0));
+    let cb0 = int0.value_field(fx, mir::Field::new(1)).load_scalar(fx);
+
+    // c + carry -> c + second intermediate carry or borrow respectively
+    let cb_in_as_u64 = fx.bcx.ins().uextend(types::I64, cb_in);
+    let cb_in_as_u64 = CValue::by_val(cb_in_as_u64, fx.layout_of(fx.tcx.types.u64));
+    let int1 = crate::num::codegen_checked_int_binop(fx, bin_op, c, cb_in_as_u64);
+    let (c, cb1) = int1.load_scalar_pair(fx);
+
+    // carry0 | carry1 -> carry or borrow respectively
+    let cb_out = fx.bcx.ins().bor(cb0, cb1);
+
+    let layout = fx.layout_of(fx.tcx.mk_tup([fx.tcx.types.u8, fx.tcx.types.u64].iter()));
+    let val = CValue::by_val_pair(cb_out, c, layout);
+    ret.write_cvalue(fx, val);
+}
diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs
index ebf98e8694b..cb1cb3c74db 100644
--- a/compiler/rustc_codegen_cranelift/src/lib.rs
+++ b/compiler/rustc_codegen_cranelift/src/lib.rs
@@ -1,4 +1,4 @@
-#![feature(rustc_private, decl_macro, never_type, hash_drain_filter, vec_into_raw_parts)]
+#![feature(rustc_private, decl_macro, never_type, hash_drain_filter, vec_into_raw_parts, once_cell)]
 #![warn(rust_2018_idioms)]
 #![warn(unused_lifetimes)]
 #![warn(unreachable_pub)]
@@ -14,7 +14,9 @@ extern crate rustc_fs_util;
 extern crate rustc_hir;
 extern crate rustc_incremental;
 extern crate rustc_index;
+extern crate rustc_interface;
 extern crate rustc_metadata;
+extern crate rustc_mir;
 extern crate rustc_session;
 extern crate rustc_span;
 extern crate rustc_target;
@@ -284,10 +286,12 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::Tar
         }
         None => {
             let mut builder =
-                cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap();
-            // Don't use "haswell" as the default, as it implies `has_lzcnt`.
-            // macOS CI is still at Ivy Bridge EP, so `lzcnt` is interpreted as `bsr`.
-            builder.enable("nehalem").unwrap();
+                cranelift_codegen::isa::lookup_variant(target_triple.clone(), variant).unwrap();
+            if target_triple.architecture == target_lexicon::Architecture::X86_64 {
+                // Don't use "haswell" as the default, as it implies `has_lzcnt`.
+                // macOS CI is still at Ivy Bridge EP, so `lzcnt` is interpreted as `bsr`.
+                builder.enable("nehalem").unwrap();
+            }
             builder
         }
     };
diff --git a/compiler/rustc_codegen_cranelift/src/optimize/mod.rs b/compiler/rustc_codegen_cranelift/src/optimize/mod.rs
index 137fb5f7731..61033d85a12 100644
--- a/compiler/rustc_codegen_cranelift/src/optimize/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/optimize/mod.rs
@@ -1,17 +1,20 @@
 //! Various optimizations specific to cg_clif
 
+use cranelift_codegen::isa::TargetIsa;
+
 use crate::prelude::*;
 
 pub(crate) mod peephole;
 
 pub(crate) fn optimize_function<'tcx>(
     tcx: TyCtxt<'tcx>,
+    isa: &dyn TargetIsa,
     instance: Instance<'tcx>,
     ctx: &mut Context,
     clif_comments: &mut crate::pretty_clif::CommentWriter,
 ) {
     // FIXME classify optimizations over opt levels once we have more
 
-    crate::pretty_clif::write_clif_file(tcx, "preopt", None, instance, &ctx, &*clif_comments);
+    crate::pretty_clif::write_clif_file(tcx, "preopt", isa, instance, &ctx, &*clif_comments);
     crate::base::verify_func(tcx, &*clif_comments, &ctx.func);
 }
diff --git a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
index cd8c5b51608..05db74745a1 100644
--- a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
+++ b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
@@ -233,7 +233,7 @@ pub(crate) fn write_ir_file(
 pub(crate) fn write_clif_file<'tcx>(
     tcx: TyCtxt<'tcx>,
     postfix: &str,
-    isa: Option<&dyn cranelift_codegen::isa::TargetIsa>,
+    isa: &dyn cranelift_codegen::isa::TargetIsa,
     instance: Instance<'tcx>,
     context: &cranelift_codegen::Context,
     mut clif_comments: &CommentWriter,
@@ -242,22 +242,23 @@ pub(crate) fn write_clif_file<'tcx>(
         tcx,
         || format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix),
         |file| {
-            let value_ranges = isa
-                .map(|isa| context.build_value_labels_ranges(isa).expect("value location ranges"));
-
             let mut clif = String::new();
             cranelift_codegen::write::decorate_function(
                 &mut clif_comments,
                 &mut clif,
                 &context.func,
-                &DisplayFunctionAnnotations { isa, value_ranges: value_ranges.as_ref() },
+                &DisplayFunctionAnnotations { isa: Some(isa), value_ranges: None },
             )
             .unwrap();
 
-            writeln!(file, "test compile")?;
-            writeln!(file, "set is_pic")?;
-            writeln!(file, "set enable_simd")?;
-            writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?;
+            for flag in isa.flags().iter() {
+                writeln!(file, "set {}", flag)?;
+            }
+            write!(file, "target {}", isa.triple().architecture.to_string())?;
+            for isa_flag in isa.isa_flags().iter() {
+                write!(file, " {}", isa_flag)?;
+            }
+            writeln!(file, "\n")?;
             writeln!(file)?;
             file.write_all(clif.as_bytes())?;
             Ok(())
diff --git a/compiler/rustc_codegen_cranelift/src/vtable.rs b/compiler/rustc_codegen_cranelift/src/vtable.rs
index 12f7092d935..4a5f9f133a2 100644
--- a/compiler/rustc_codegen_cranelift/src/vtable.rs
+++ b/compiler/rustc_codegen_cranelift/src/vtable.rs
@@ -1,10 +1,9 @@
 //! Codegen vtables and vtable accesses.
 //!
 //! See `rustc_codegen_ssa/src/meth.rs` for reference.
-// FIXME dedup this logic between miri, cg_llvm and cg_clif
 
+use crate::constant::data_id_for_alloc_id;
 use crate::prelude::*;
-use super::constant::pointer_for_allocation;
 
 fn vtable_memflags() -> MemFlags {
     let mut flags = MemFlags::trusted(); // A vtable access is always aligned and will never trap.
@@ -38,7 +37,7 @@ pub(crate) fn min_align_of_obj(fx: &mut FunctionCx<'_, '_, '_>, vtable: Value) -
         pointer_ty(fx.tcx),
         vtable_memflags(),
         vtable,
-        (ty::COMMON_VTABLE_ENTRIES_SIZE * usize_size) as i32,
+        (ty::COMMON_VTABLE_ENTRIES_ALIGN * usize_size) as i32,
     )
 }
 
@@ -69,16 +68,12 @@ pub(crate) fn get_vtable<'tcx>(
     ty: Ty<'tcx>,
     trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
 ) -> Value {
-    let vtable_ptr = if let Some(vtable_ptr) = fx.vtables.get(&(ty, trait_ref)) {
-        *vtable_ptr
-    } else {
-        let vtable_alloc_id = fx.tcx.vtable_allocation(ty, trait_ref);
-        let vtable_allocation = fx.tcx.global_alloc(vtable_alloc_id).unwrap_memory();
-        let vtable_ptr = pointer_for_allocation(fx, vtable_allocation);
-
-        fx.vtables.insert((ty, trait_ref), vtable_ptr);
-        vtable_ptr
-    };
-
-    vtable_ptr.get_addr(fx)
+    let alloc_id = fx.tcx.vtable_allocation(ty, trait_ref);
+    let data_id =
+        data_id_for_alloc_id(&mut fx.constants_cx, &mut *fx.module, alloc_id, Mutability::Not);
+    let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
+    if fx.clif_comments.enabled() {
+        fx.add_comment(local_data_id, format!("vtable: {:?}", alloc_id));
+    }
+    fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
 }