Auto merge of #83637 - bjorn3:sync_cg_clif-2021-03-29, r=bjorn3

Sync rustc_codegen_cranelift

The main highlight of this sync is support for cross-compiling to Windows using MinGW. Native compilation with MinGW would also work I think, but using the MSVC toolchain is not yet supported as PE TLS is not yet implemented. Another nice improvement is that crate metadata is now loaded using mmap instead of by reading files. This improves compilation time a bit.

r? `@ghost`

`@rustbot` label +A-codegen +A-cranelift +T-compiler

26 files changed, 433 insertions, 266 deletions

diff --git a/compiler/rustc_codegen_cranelift/src/abi/comments.rs b/compiler/rustc_codegen_cranelift/src/abi/comments.rs
index c3cf90e1e70..5fbaed7283a 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/comments.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/comments.rs
@@ -11,9 +11,11 @@ use cranelift_codegen::entity::EntityRef;
 use crate::prelude::*;
 
 pub(super) fn add_args_header_comment(fx: &mut FunctionCx<'_, '_, '_>) {
-    fx.add_global_comment(
-        "kind  loc.idx   param    pass mode                            ty".to_string(),
-    );
+    if fx.clif_comments.enabled() {
+        fx.add_global_comment(
+            "kind  loc.idx   param    pass mode                            ty".to_string(),
+        );
+    }
 }
 
 pub(super) fn add_arg_comment<'tcx>(
@@ -25,6 +27,10 @@ pub(super) fn add_arg_comment<'tcx>(
     arg_abi_mode: PassMode,
     arg_layout: TyAndLayout<'tcx>,
 ) {
+    if !fx.clif_comments.enabled() {
+        return;
+    }
+
     let local = if let Some(local) = local {
         Cow::Owned(format!("{:?}", local))
     } else {
@@ -59,10 +65,12 @@ pub(super) fn add_arg_comment<'tcx>(
 }
 
 pub(super) fn add_locals_header_comment(fx: &mut FunctionCx<'_, '_, '_>) {
-    fx.add_global_comment(String::new());
-    fx.add_global_comment(
-        "kind  local ty                              size align (abi,pref)".to_string(),
-    );
+    if fx.clif_comments.enabled() {
+        fx.add_global_comment(String::new());
+        fx.add_global_comment(
+            "kind  local ty                              size align (abi,pref)".to_string(),
+        );
+    }
 }
 
 pub(super) fn add_local_place_comments<'tcx>(
@@ -70,6 +78,9 @@ pub(super) fn add_local_place_comments<'tcx>(
     place: CPlace<'tcx>,
     local: Local,
 ) {
+    if !fx.clif_comments.enabled() {
+        return;
+    }
     let TyAndLayout { ty, layout } = place.layout();
     let rustc_target::abi::Layout { size, align, abi: _, variants: _, fields: _, largest_niche: _ } =
         layout;
@@ -90,7 +101,7 @@ pub(super) fn add_local_place_comments<'tcx>(
             } else {
                 Cow::Borrowed("")
             };
-            match ptr.base_and_offset() {
+            match ptr.debug_base_and_offset() {
                 (crate::pointer::PointerBase::Addr(addr), offset) => {
                     ("reuse", format!("storage={}{}{}", addr, offset, meta).into())
                 }
diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
index b158d73f3a1..0e7829eaa26 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
@@ -1,6 +1,5 @@
 //! Handling of everything related to the calling convention. Also fills `fx.local_map`.
 
-#[cfg(debug_assertions)]
 mod comments;
 mod pass_mode;
 mod returning;
@@ -75,8 +74,9 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         let func_id = import_function(self.tcx, self.cx.module, inst);
         let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func);
 
-        #[cfg(debug_assertions)]
-        self.add_comment(func_ref, format!("{:?}", inst));
+        if self.clif_comments.enabled() {
+            self.add_comment(func_ref, format!("{:?}", inst));
+        }
 
         func_ref
     }
@@ -92,8 +92,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         let func_id = self.cx.module.declare_function(&name, Linkage::Import, &sig).unwrap();
         let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func);
         let call_inst = self.bcx.ins().call(func_ref, args);
-        #[cfg(debug_assertions)]
-        {
+        if self.clif_comments.enabled() {
             self.add_comment(call_inst, format!("easy_call {}", name));
         }
         let results = self.bcx.inst_results(call_inst);
@@ -149,7 +148,6 @@ fn make_local_place<'tcx>(
         CPlace::new_stack_slot(fx, layout)
     };
 
-    #[cfg(debug_assertions)]
     self::comments::add_local_place_comments(fx, place, local);
 
     place
@@ -163,7 +161,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
 
     let ssa_analyzed = crate::analyze::analyze(fx);
 
-    #[cfg(debug_assertions)]
     self::comments::add_args_header_comment(fx);
 
     let mut block_params_iter = fx.bcx.func.dfg.block_params(start_block).to_vec().into_iter();
@@ -228,7 +225,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
     fx.fn_abi = Some(fn_abi);
     assert!(block_params_iter.next().is_none(), "arg_value left behind");
 
-    #[cfg(debug_assertions)]
     self::comments::add_locals_header_comment(fx);
 
     for (local, arg_kind, ty) in func_params {
@@ -256,7 +252,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_
                         CPlace::for_ptr(addr, val.layout())
                     };
 
-                    #[cfg(debug_assertions)]
                     self::comments::add_local_place_comments(fx, place, local);
 
                     assert_eq!(fx.local_map.push(place), local);
@@ -392,8 +387,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
     let (func_ref, first_arg) = match instance {
         // Trait object call
         Some(Instance { def: InstanceDef::Virtual(_, idx), .. }) => {
-            #[cfg(debug_assertions)]
-            {
+            if fx.clif_comments.enabled() {
                 let nop_inst = fx.bcx.ins().nop();
                 fx.add_comment(
                     nop_inst,
@@ -414,8 +408,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
 
         // Indirect call
         None => {
-            #[cfg(debug_assertions)]
-            {
+            if fx.clif_comments.enabled() {
                 let nop_inst = fx.bcx.ins().nop();
                 fx.add_comment(nop_inst, "indirect call");
             }
@@ -477,10 +470,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
     // FIXME find a cleaner way to support varargs
     if fn_sig.c_variadic {
         if !matches!(fn_sig.abi, Abi::C { .. }) {
-            fx.tcx.sess.span_fatal(
-                span,
-                &format!("Variadic call for non-C abi {:?}", fn_sig.abi),
-            );
+            fx.tcx.sess.span_fatal(span, &format!("Variadic call for non-C abi {:?}", fn_sig.abi));
         }
         let sig_ref = fx.bcx.func.dfg.call_signature(call_inst).unwrap();
         let abi_params = call_args
diff --git a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
index d58f952f53c..7c275965199 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
@@ -208,7 +208,7 @@ pub(super) fn from_casted_value<'tcx>(
     });
     let ptr = Pointer::new(fx.bcx.ins().stack_addr(pointer_ty(fx.tcx), stack_slot, 0));
     let mut offset = 0;
-    let mut block_params_iter = block_params.into_iter().copied();
+    let mut block_params_iter = block_params.iter().copied();
     for param in abi_params {
         let val = ptr.offset_i64(fx, offset).store(
             fx,
@@ -248,8 +248,8 @@ pub(super) fn adjust_arg_for_abi<'tcx>(
 /// as necessary.
 pub(super) fn cvalue_for_param<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
-    #[cfg_attr(not(debug_assertions), allow(unused_variables))] local: Option<mir::Local>,
-    #[cfg_attr(not(debug_assertions), allow(unused_variables))] local_field: Option<usize>,
+    local: Option<mir::Local>,
+    local_field: Option<usize>,
     arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
     block_params_iter: &mut impl Iterator<Item = Value>,
 ) -> Option<CValue<'tcx>> {
@@ -263,7 +263,6 @@ pub(super) fn cvalue_for_param<'tcx>(
         })
         .collect::<SmallVec<[_; 2]>>();
 
-    #[cfg(debug_assertions)]
     crate::abi::comments::add_arg_comment(
         fx,
         "arg",
diff --git a/compiler/rustc_codegen_cranelift/src/abi/returning.rs b/compiler/rustc_codegen_cranelift/src/abi/returning.rs
index 9fa066df69b..e1c53224b4f 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/returning.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/returning.rs
@@ -84,10 +84,6 @@ pub(super) fn codegen_return_param<'tcx>(
         }
     };
 
-    #[cfg(not(debug_assertions))]
-    let _ = ret_param;
-
-    #[cfg(debug_assertions)]
     crate::abi::comments::add_arg_comment(
         fx,
         "ret",
@@ -146,7 +142,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx, T>(
                 let results = fx
                     .bcx
                     .inst_results(call_inst)
-                    .into_iter()
+                    .iter()
                     .copied()
                     .collect::<SmallVec<[Value; 2]>>();
                 let result =
diff --git a/compiler/rustc_codegen_cranelift/src/allocator.rs b/compiler/rustc_codegen_cranelift/src/allocator.rs
index efb64233ef2..f60645a9f97 100644
--- a/compiler/rustc_codegen_cranelift/src/allocator.rs
+++ b/compiler/rustc_codegen_cranelift/src/allocator.rs
@@ -3,6 +3,7 @@
 
 use crate::prelude::*;
 
+use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_ast::expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS};
 use rustc_span::symbol::sym;
 
@@ -92,7 +93,7 @@ fn codegen_inner(
             bcx.finalize();
         }
         module
-            .define_function(func_id, &mut ctx, &mut cranelift_codegen::binemit::NullTrapSink {})
+            .define_function(func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {})
             .unwrap();
         unwind_context.add_function(func_id, &ctx, module.isa());
     }
@@ -132,7 +133,7 @@ fn codegen_inner(
         bcx.finalize();
     }
     module
-        .define_function(func_id, &mut ctx, &mut cranelift_codegen::binemit::NullTrapSink {})
+        .define_function(func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {})
         .unwrap();
     unwind_context.add_function(func_id, &ctx, module.isa());
 }
diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs
index 8b5ae9e0541..b34a29c25b9 100644
--- a/compiler/rustc_codegen_cranelift/src/base.rs
+++ b/compiler/rustc_codegen_cranelift/src/base.rs
@@ -1,5 +1,6 @@
 //! Codegen of a single function
 
+use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_index::vec::IndexVec;
 use rustc_middle::ty::adjustment::PointerCast;
 use rustc_middle::ty::layout::FnAbiExt;
@@ -7,11 +8,7 @@ use rustc_target::abi::call::FnAbi;
 
 use crate::prelude::*;
 
-pub(crate) fn codegen_fn<'tcx>(
-    cx: &mut crate::CodegenCx<'_, 'tcx>,
-    instance: Instance<'tcx>,
-    linkage: Linkage,
-) {
+pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) {
     let tcx = cx.tcx;
 
     let _inst_guard =
@@ -23,7 +20,7 @@ pub(crate) fn codegen_fn<'tcx>(
     // Declare function
     let name = tcx.symbol_name(instance).name.to_string();
     let sig = get_function_sig(tcx, cx.module.isa().triple(), instance);
-    let func_id = cx.module.declare_function(&name, linkage, &sig).unwrap();
+    let func_id = cx.module.declare_function(&name, Linkage::Local, &sig).unwrap();
 
     cx.cached_context.clear();
 
@@ -131,7 +128,7 @@ pub(crate) fn codegen_fn<'tcx>(
     let module = &mut cx.module;
     tcx.sess.time("define function", || {
         module
-            .define_function(func_id, context, &mut cranelift_codegen::binemit::NullTrapSink {})
+            .define_function(func_id, context, &mut NullTrapSink {}, &mut NullStackMapSink {})
             .unwrap()
     });
 
@@ -219,8 +216,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
             codegen_stmt(fx, block, stmt);
         }
 
-        #[cfg(debug_assertions)]
-        {
+        if fx.clif_comments.enabled() {
             let mut terminator_head = "\n".to_string();
             bb_data.terminator().kind.fmt_head(&mut terminator_head).unwrap();
             let inst = fx.bcx.func.layout.last_inst(block).unwrap();
@@ -433,12 +429,14 @@ fn codegen_stmt<'tcx>(
 
     fx.set_debug_loc(stmt.source_info);
 
-    #[cfg(false_debug_assertions)]
+    #[cfg(disabled)]
     match &stmt.kind {
         StatementKind::StorageLive(..) | StatementKind::StorageDead(..) => {} // Those are not very useful
         _ => {
-            let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap();
-            fx.add_comment(inst, format!("{:?}", stmt));
+            if fx.clif_comments.enabled() {
+                let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap();
+                fx.add_comment(inst, format!("{:?}", stmt));
+            }
         }
     }
 
@@ -464,16 +462,16 @@ fn codegen_stmt<'tcx>(
                     let val = crate::constant::codegen_tls_ref(fx, def_id, lval.layout());
                     lval.write_cvalue(fx, val);
                 }
-                Rvalue::BinaryOp(bin_op, box (ref lhs, ref rhs)) => {
-                    let lhs = codegen_operand(fx, lhs);
-                    let rhs = codegen_operand(fx, rhs);
+                Rvalue::BinaryOp(bin_op, ref lhs_rhs) => {
+                    let lhs = codegen_operand(fx, &lhs_rhs.0);
+                    let rhs = codegen_operand(fx, &lhs_rhs.1);
 
                     let res = crate::num::codegen_binop(fx, bin_op, lhs, rhs);
                     lval.write_cvalue(fx, res);
                 }
-                Rvalue::CheckedBinaryOp(bin_op, box (ref lhs, ref rhs)) => {
-                    let lhs = codegen_operand(fx, lhs);
-                    let rhs = codegen_operand(fx, rhs);
+                Rvalue::CheckedBinaryOp(bin_op, ref lhs_rhs) => {
+                    let lhs = codegen_operand(fx, &lhs_rhs.0);
+                    let rhs = codegen_operand(fx, &lhs_rhs.1);
 
                     let res = if !fx.tcx.sess.overflow_checks() {
                         let val =
@@ -659,7 +657,9 @@ fn codegen_stmt<'tcx>(
                         .val
                         .try_to_bits(fx.tcx.data_layout.pointer_size)
                         .unwrap();
-                    if fx.clif_type(operand.layout().ty) == Some(types::I8) {
+                    if operand.layout().size.bytes() == 0 {
+                        // Do nothing for ZST's
+                    } else if fx.clif_type(operand.layout().ty) == Some(types::I8) {
                         let times = fx.bcx.ins().iconst(fx.pointer_type, times as i64);
                         // FIXME use emit_small_memset where possible
                         let addr = lval.to_ptr().get_addr(fx);
@@ -832,25 +832,18 @@ fn codegen_stmt<'tcx>(
             }
         }
         StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"),
-        StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
-          src,
-          dst,
-          count,
-        }) => {
-            let dst = codegen_operand(fx, dst);
+        StatementKind::CopyNonOverlapping(inner) => {
+            let dst = codegen_operand(fx, &inner.dst);
             let pointee = dst
-              .layout()
-              .pointee_info_at(fx, rustc_target::abi::Size::ZERO)
-              .expect("Expected pointer");
+                .layout()
+                .pointee_info_at(fx, rustc_target::abi::Size::ZERO)
+                .expect("Expected pointer");
             let dst = dst.load_scalar(fx);
-            let src = codegen_operand(fx, src).load_scalar(fx);
-            let count = codegen_operand(fx, count).load_scalar(fx);
+            let src = codegen_operand(fx, &inner.src).load_scalar(fx);
+            let count = codegen_operand(fx, &inner.count).load_scalar(fx);
             let elem_size: u64 = pointee.size.bytes();
-            let bytes = if elem_size != 1 {
-               fx.bcx.ins().imul_imm(count, elem_size as i64)
-            } else {
-               count
-            };
+            let bytes =
+                if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
             fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, bytes);
         }
     }
diff --git a/compiler/rustc_codegen_cranelift/src/codegen_i128.rs b/compiler/rustc_codegen_cranelift/src/codegen_i128.rs
index ae75e6508cb..ffe1922ab90 100644
--- a/compiler/rustc_codegen_cranelift/src/codegen_i128.rs
+++ b/compiler/rustc_codegen_cranelift/src/codegen_i128.rs
@@ -32,18 +32,56 @@ pub(crate) fn maybe_codegen<'tcx>(
         BinOp::Add | BinOp::Sub if !checked => None,
         BinOp::Mul if !checked => {
             let val_ty = if is_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 };
-            Some(fx.easy_call("__multi3", &[lhs, rhs], val_ty))
+            if fx.tcx.sess.target.is_like_windows {
+                let ret_place = CPlace::new_stack_slot(fx, lhs.layout());
+                let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
+                let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
+                assert!(lhs_extra.is_none());
+                assert!(rhs_extra.is_none());
+                let args =
+                    [ret_place.to_ptr().get_addr(fx), lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)];
+                fx.lib_call(
+                    "__multi3",
+                    vec![
+                        AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
+                        AbiParam::new(pointer_ty(fx.tcx)),
+                        AbiParam::new(pointer_ty(fx.tcx)),
+                    ],
+                    vec![],
+                    &args,
+                );
+                Some(ret_place.to_cvalue(fx))
+            } else {
+                Some(fx.easy_call("__multi3", &[lhs, rhs], val_ty))
+            }
         }
         BinOp::Add | BinOp::Sub | BinOp::Mul => {
             assert!(checked);
             let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter());
             let out_place = CPlace::new_stack_slot(fx, fx.layout_of(out_ty));
-            let param_types = vec![
-                AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
-                AbiParam::new(types::I128),
-                AbiParam::new(types::I128),
-            ];
-            let args = [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)];
+            let (param_types, args) = if fx.tcx.sess.target.is_like_windows {
+                let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
+                let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
+                assert!(lhs_extra.is_none());
+                assert!(rhs_extra.is_none());
+                (
+                    vec![
+                        AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
+                        AbiParam::new(pointer_ty(fx.tcx)),
+                        AbiParam::new(pointer_ty(fx.tcx)),
+                    ],
+                    [out_place.to_ptr().get_addr(fx), lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)],
+                )
+            } else {
+                (
+                    vec![
+                        AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn),
+                        AbiParam::new(types::I128),
+                        AbiParam::new(types::I128),
+                    ],
+                    [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)],
+                )
+            };
             let name = match (bin_op, is_signed) {
                 (BinOp::Add, false) => "__rust_u128_addo",
                 (BinOp::Add, true) => "__rust_i128_addo",
@@ -57,20 +95,33 @@ pub(crate) fn maybe_codegen<'tcx>(
             Some(out_place.to_cvalue(fx))
         }
         BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"),
-        BinOp::Div => {
+        BinOp::Div | BinOp::Rem => {
             assert!(!checked);
-            if is_signed {
-                Some(fx.easy_call("__divti3", &[lhs, rhs], fx.tcx.types.i128))
-            } else {
-                Some(fx.easy_call("__udivti3", &[lhs, rhs], fx.tcx.types.u128))
-            }
-        }
-        BinOp::Rem => {
-            assert!(!checked);
-            if is_signed {
-                Some(fx.easy_call("__modti3", &[lhs, rhs], fx.tcx.types.i128))
+            let name = match (bin_op, is_signed) {
+                (BinOp::Div, false) => "__udivti3",
+                (BinOp::Div, true) => "__divti3",
+                (BinOp::Rem, false) => "__umodti3",
+                (BinOp::Rem, true) => "__modti3",
+                _ => unreachable!(),
+            };
+            if fx.tcx.sess.target.is_like_windows {
+                let (lhs_ptr, lhs_extra) = lhs.force_stack(fx);
+                let (rhs_ptr, rhs_extra) = rhs.force_stack(fx);
+                assert!(lhs_extra.is_none());
+                assert!(rhs_extra.is_none());
+                let args = [lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)];
+                let ret = fx.lib_call(
+                    name,
+                    vec![AbiParam::new(pointer_ty(fx.tcx)), AbiParam::new(pointer_ty(fx.tcx))],
+                    vec![AbiParam::new(types::I64X2)],
+                    &args,
+                )[0];
+                // FIXME use bitcast instead of store to get from i64x2 to i128
+                let ret_place = CPlace::new_stack_slot(fx, lhs.layout());
+                ret_place.to_ptr().store(fx, ret, MemFlags::trusted());
+                Some(ret_place.to_cvalue(fx))
             } else {
-                Some(fx.easy_call("__umodti3", &[lhs, rhs], fx.tcx.types.u128))
+                Some(fx.easy_call(name, &[lhs, rhs], lhs.layout().ty))
             }
         }
         BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => {
diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs
index 6a4a6744a5c..b5874f62535 100644
--- a/compiler/rustc_codegen_cranelift/src/common.rs
+++ b/compiler/rustc_codegen_cranelift/src/common.rs
@@ -361,8 +361,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         let _ = self.cx.module.define_data(msg_id, &data_ctx);
 
         let local_msg_id = self.cx.module.declare_data_in_func(msg_id, self.bcx.func);
-        #[cfg(debug_assertions)]
-        {
+        if self.clif_comments.enabled() {
             self.add_comment(local_msg_id, msg);
         }
         self.bcx.ins().global_value(self.pointer_type, local_msg_id)
diff --git a/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs b/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs
new file mode 100644
index 00000000000..177f850afb3
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs
@@ -0,0 +1,41 @@
+macro builtin_functions($register:ident; $(fn $name:ident($($arg_name:ident: $arg_ty:ty),*) -> $ret_ty:ty;)*) {
+    #[cfg(feature = "jit")]
+    #[allow(improper_ctypes)]
+    extern "C" {
+        $(fn $name($($arg_name: $arg_ty),*) -> $ret_ty;)*
+    }
+
+    #[cfg(feature = "jit")]
+    pub(crate) fn $register(builder: &mut cranelift_jit::JITBuilder) {
+        for &(name, val) in &[$((stringify!($name), $name as *const u8)),*] {
+            builder.symbol(name, val);
+        }
+    }
+}
+
+builtin_functions! {
+    register_functions_for_jit;
+
+    // integers
+    fn __multi3(a: i128, b: i128) -> i128;
+    fn __udivti3(n: u128, d: u128) -> u128;
+    fn __divti3(n: i128, d: i128) -> i128;
+    fn __umodti3(n: u128, d: u128) -> u128;
+    fn __modti3(n: i128, d: i128) -> i128;
+    fn __rust_u128_addo(a: u128, b: u128) -> (u128, bool);
+    fn __rust_i128_addo(a: i128, b: i128) -> (i128, bool);
+    fn __rust_u128_subo(a: u128, b: u128) -> (u128, bool);
+    fn __rust_i128_subo(a: i128, b: i128) -> (i128, bool);
+    fn __rust_u128_mulo(a: u128, b: u128) -> (u128, bool);
+    fn __rust_i128_mulo(a: i128, b: i128) -> (i128, bool);
+
+    // floats
+    fn __floattisf(i: i128) -> f32;
+    fn __floattidf(i: i128) -> f64;
+    fn __floatuntisf(i: u128) -> f32;
+    fn __floatuntidf(i: u128) -> f64;
+    fn __fixsfti(f: f32) -> i128;
+    fn __fixdfti(f: f64) -> i128;
+    fn __fixunssfti(f: f32) -> u128;
+    fn __fixunsdfti(f: f64) -> u128;
+}
diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs
index f4cbfb6967f..fcd41c84465 100644
--- a/compiler/rustc_codegen_cranelift/src/constant.rs
+++ b/compiler/rustc_codegen_cranelift/src/constant.rs
@@ -85,8 +85,9 @@ pub(crate) fn codegen_tls_ref<'tcx>(
 ) -> CValue<'tcx> {
     let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
     let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
-    fx.add_comment(local_data_id, format!("tls {:?}", def_id));
+    if fx.clif_comments.enabled() {
+        fx.add_comment(local_data_id, format!("tls {:?}", def_id));
+    }
     let tls_ptr = fx.bcx.ins().tls_value(fx.pointer_type, local_data_id);
     CValue::by_val(tls_ptr, layout)
 }
@@ -98,8 +99,9 @@ fn codegen_static_ref<'tcx>(
 ) -> CPlace<'tcx> {
     let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
     let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
-    fx.add_comment(local_data_id, format!("{:?}", def_id));
+    if fx.clif_comments.enabled() {
+        fx.add_comment(local_data_id, format!("{:?}", def_id));
+    }
     let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id);
     assert!(!layout.is_unsized(), "unsized statics aren't supported");
     assert!(
@@ -122,7 +124,9 @@ pub(crate) fn codegen_constant<'tcx>(
     };
     let const_val = match const_.val {
         ConstKind::Value(const_val) => const_val,
-        ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted }) if fx.tcx.is_static(def.did) => {
+        ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted })
+            if fx.tcx.is_static(def.did) =>
+        {
             assert!(substs.is_empty());
             assert!(promoted.is_none());
 
@@ -183,8 +187,9 @@ pub(crate) fn codegen_const_value<'tcx>(
                                 data_id_for_alloc_id(fx.cx.module, ptr.alloc_id, alloc.mutability);
                             let local_data_id =
                                 fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-                            #[cfg(debug_assertions)]
-                            fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id));
+                            if fx.clif_comments.enabled() {
+                                fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id));
+                            }
                             fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
                         }
                         Some(GlobalAlloc::Function(instance)) => {
@@ -199,8 +204,9 @@ pub(crate) fn codegen_const_value<'tcx>(
                             let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false);
                             let local_data_id =
                                 fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-                            #[cfg(debug_assertions)]
-                            fx.add_comment(local_data_id, format!("{:?}", def_id));
+                            if fx.clif_comments.enabled() {
+                                fx.add_comment(local_data_id, format!("{:?}", def_id));
+                            }
                             fx.bcx.ins().global_value(fx.pointer_type, local_data_id)
                         }
                         None => bug!("missing allocation {:?}", ptr.alloc_id),
@@ -241,8 +247,9 @@ fn pointer_for_allocation<'tcx>(
     let data_id = data_id_for_alloc_id(fx.cx.module, alloc_id, alloc.mutability);
 
     let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
-    fx.add_comment(local_data_id, format!("{:?}", alloc_id));
+    if fx.clif_comments.enabled() {
+        fx.add_comment(local_data_id, format!("{:?}", alloc_id));
+    }
     let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id);
     crate::pointer::Pointer::new(global_ptr)
 }
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
index 30ed356c762..8578ab33ced 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
@@ -39,11 +39,11 @@ fn osstr_as_utf8_bytes(path: &OsStr) -> &[u8] {
     #[cfg(unix)]
     {
         use std::os::unix::ffi::OsStrExt;
-        return path.as_bytes();
+        path.as_bytes()
     }
     #[cfg(not(unix))]
     {
-        return path.to_str().unwrap().as_bytes();
+        path.to_str().unwrap().as_bytes()
     }
 }
 
diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs
index b87dcc41928..ed3bdedddce 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs
@@ -119,11 +119,10 @@ fn module_codegen(
         tcx.sess.opts.debuginfo != DebugInfo::None,
     );
     super::predefine_mono_items(&mut cx, &mono_items);
-    for (mono_item, (linkage, visibility)) in mono_items {
-        let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
+    for (mono_item, _) in mono_items {
         match mono_item {
             MonoItem::Fn(inst) => {
-                cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst, linkage));
+                cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst));
             }
             MonoItem::Static(def_id) => {
                 crate::constant::codegen_static(&mut cx.constants_cx, def_id)
@@ -163,6 +162,21 @@ pub(super) fn run_aot(
     metadata: EncodedMetadata,
     need_metadata_module: bool,
 ) -> Box<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)> {
+    use rustc_span::symbol::sym;
+
+    let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID);
+    let subsystem = tcx.sess.first_attr_value_str_by_name(crate_attrs, sym::windows_subsystem);
+    let windows_subsystem = subsystem.map(|subsystem| {
+        if subsystem != sym::windows && subsystem != sym::console {
+            tcx.sess.fatal(&format!(
+                "invalid windows subsystem `{}`, only \
+                                    `windows` and `console` are allowed",
+                subsystem
+            ));
+        }
+        subsystem.to_string()
+    });
+
     let mut work_products = FxHashMap::default();
 
     let cgus = if tcx.sess.opts.output_types.should_codegen() {
@@ -280,7 +294,7 @@ pub(super) fn run_aot(
             allocator_module,
             metadata_module,
             metadata,
-            windows_subsystem: None, // Windows is not yet supported
+            windows_subsystem,
             linker_info: LinkerInfo::new(tcx),
             crate_info: CrateInfo::new(tcx),
         },
diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
index 245df03ffb8..dbe1ff083f0 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
@@ -5,8 +5,10 @@ use std::cell::RefCell;
 use std::ffi::CString;
 use std::os::raw::{c_char, c_int};
 
+use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_codegen_ssa::CrateInfo;
 use rustc_middle::mir::mono::MonoItem;
+use rustc_session::config::EntryFnType;
 
 use cranelift_jit::{JITBuilder, JITModule};
 
@@ -28,20 +30,11 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
     let mut jit_builder =
         JITBuilder::with_isa(crate::build_isa(tcx.sess), cranelift_module::default_libcall_names());
     jit_builder.hotswap(matches!(backend_config.codegen_mode, CodegenMode::JitLazy));
+    crate::compiler_builtins::register_functions_for_jit(&mut jit_builder);
     jit_builder.symbols(imported_symbols);
     let mut jit_module = JITModule::new(jit_builder);
     assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type());
 
-    let sig = Signature {
-        params: vec![
-            AbiParam::new(jit_module.target_config().pointer_type()),
-            AbiParam::new(jit_module.target_config().pointer_type()),
-        ],
-        returns: vec![AbiParam::new(jit_module.target_config().pointer_type() /*isize*/)],
-        call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
-    };
-    let main_func_id = jit_module.declare_function("main", Linkage::Import, &sig).unwrap();
-
     let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE);
     let mono_items = cgus
         .iter()
@@ -55,15 +48,12 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
 
     super::time(tcx, "codegen mono items", || {
         super::predefine_mono_items(&mut cx, &mono_items);
-        for (mono_item, (linkage, visibility)) in mono_items {
-            let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
+        for (mono_item, _) in mono_items {
             match mono_item {
                 MonoItem::Fn(inst) => match backend_config.codegen_mode {
                     CodegenMode::Aot => unreachable!(),
                     CodegenMode::Jit => {
-                        cx.tcx
-                            .sess
-                            .time("codegen fn", || crate::base::codegen_fn(&mut cx, inst, linkage));
+                        cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst));
                     }
                     CodegenMode::JitLazy => codegen_shim(&mut cx, inst),
                 },
@@ -86,24 +76,17 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
         tcx.sess.fatal("Inline asm is not supported in JIT mode");
     }
 
-    crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context);
     crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context);
 
     tcx.sess.abort_if_errors();
 
     jit_module.finalize_definitions();
-
     let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_module) };
 
-    let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id);
-
     println!(
         "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed"
     );
 
-    let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
-        unsafe { ::std::mem::transmute(finalized_main) };
-
     let args = ::std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new());
     let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string())
         .chain(args.split(' '))
@@ -118,12 +101,58 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
     BACKEND_CONFIG.with(|tls_backend_config| {
         assert!(tls_backend_config.borrow_mut().replace(backend_config).is_none())
     });
-    CURRENT_MODULE
-        .with(|current_module| assert!(current_module.borrow_mut().replace(jit_module).is_none()));
 
-    let ret = f(args.len() as c_int, argv.as_ptr());
+    let (main_def_id, entry_ty) = tcx.entry_fn(LOCAL_CRATE).unwrap();
+    let instance = Instance::mono(tcx, main_def_id.to_def_id()).polymorphize(tcx);
+
+    match entry_ty {
+        EntryFnType::Main => {
+            // FIXME set program arguments somehow
 
-    std::process::exit(ret);
+            let main_sig = Signature {
+                params: vec![],
+                returns: vec![],
+                call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
+            };
+            let main_func_id = jit_module
+                .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &main_sig)
+                .unwrap();
+            let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id);
+
+            CURRENT_MODULE.with(|current_module| {
+                assert!(current_module.borrow_mut().replace(jit_module).is_none())
+            });
+
+            let f: extern "C" fn() = unsafe { ::std::mem::transmute(finalized_main) };
+            f();
+            std::process::exit(0);
+        }
+        EntryFnType::Start => {
+            let start_sig = Signature {
+                params: vec![
+                    AbiParam::new(jit_module.target_config().pointer_type()),
+                    AbiParam::new(jit_module.target_config().pointer_type()),
+                ],
+                returns: vec![AbiParam::new(
+                    jit_module.target_config().pointer_type(), /*isize*/
+                )],
+                call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)),
+            };
+            let start_func_id = jit_module
+                .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &start_sig)
+                .unwrap();
+            let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id);
+
+            CURRENT_MODULE.with(|current_module| {
+                assert!(current_module.borrow_mut().replace(jit_module).is_none())
+            });
+
+            let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
+                unsafe { ::std::mem::transmute(finalized_start) };
+            let ret = f(args.len() as c_int, argv.as_ptr());
+            std::process::exit(ret);
+        }
+    }
 }
 
 #[no_mangle]
@@ -144,8 +173,7 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8
             jit_module.prepare_for_function_redefine(func_id).unwrap();
 
             let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module, false);
-            tcx.sess
-                .time("codegen fn", || crate::base::codegen_fn(&mut cx, instance, Linkage::Export));
+            tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, instance));
 
             let (global_asm, _debug_context, unwind_context) = cx.finalize();
             assert!(global_asm.is_empty());
@@ -220,7 +248,7 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> {
     imported_symbols
 }
 
-pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) {
+fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) {
     let tcx = cx.tcx;
 
     let pointer_type = cx.module.target_config().pointer_type();
@@ -267,7 +295,8 @@ pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'t
         .define_function(
             func_id,
             &mut Context::for_function(trampoline),
-            &mut cranelift_codegen::binemit::NullTrapSink {},
+            &mut NullTrapSink {},
+            &mut NullStackMapSink {},
         )
         .unwrap();
 }
diff --git a/compiler/rustc_codegen_cranelift/src/driver/mod.rs b/compiler/rustc_codegen_cranelift/src/driver/mod.rs
index b994f28ffef..d49182a07b7 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/mod.rs
@@ -44,13 +44,19 @@ fn predefine_mono_items<'tcx>(
     mono_items: &[(MonoItem<'tcx>, (RLinkage, Visibility))],
 ) {
     cx.tcx.sess.time("predefine functions", || {
+        let is_compiler_builtins = cx.tcx.is_compiler_builtins(LOCAL_CRATE);
         for &(mono_item, (linkage, visibility)) in mono_items {
             match mono_item {
                 MonoItem::Fn(instance) => {
                     let name = cx.tcx.symbol_name(instance).name.to_string();
                     let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, name));
                     let sig = get_function_sig(cx.tcx, cx.module.isa().triple(), instance);
-                    let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility);
+                    let linkage = crate::linkage::get_clif_linkage(
+                        mono_item,
+                        linkage,
+                        visibility,
+                        is_compiler_builtins,
+                    );
                     cx.module.declare_function(&name, linkage, &sig).unwrap();
                 }
                 MonoItem::Static(_) | MonoItem::GlobalAsm(_) => {}
diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
index 5b3df2bd382..1fb5e86aed7 100644
--- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs
+++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
@@ -20,6 +20,10 @@ pub(crate) fn codegen_inline_asm<'tcx>(
     if template.is_empty() {
         // Black box
         return;
+    } else if template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string()) {
+        let true_ = fx.bcx.ins().iconst(types::I32, 1);
+        fx.bcx.ins().trapnz(true_, TrapCode::User(1));
+        return;
     }
 
     let mut slot_size = Size::from_bytes(0);
@@ -193,8 +197,9 @@ fn call_inline_asm<'tcx>(
         offset: None,
         size: u32::try_from(slot_size.bytes()).unwrap(),
     });
-    #[cfg(debug_assertions)]
-    fx.add_comment(stack_slot, "inline asm scratch slot");
+    if fx.clif_comments.enabled() {
+        fx.add_comment(stack_slot, "inline asm scratch slot");
+    }
 
     let inline_asm_func = fx
         .cx
@@ -210,8 +215,9 @@ fn call_inline_asm<'tcx>(
         )
         .unwrap();
     let inline_asm_func = fx.cx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
-    fx.add_comment(inline_asm_func, asm_name);
+    if fx.clif_comments.enabled() {
+        fx.add_comment(inline_asm_func, asm_name);
+    }
 
     for (_reg, offset, value) in inputs {
         fx.bcx.ins().stack_store(value, stack_slot, i32::try_from(offset.bytes()).unwrap());
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
index 86df71a0dfc..c7ce32b385e 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
@@ -88,7 +88,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 let idx_bytes = match idx_const {
                     ConstValue::ByRef { alloc, offset } => {
                         let ptr = Pointer::new(AllocId(0 /* dummy */), offset);
-                        let size = Size::from_bytes(4 * u64::from(ret_lane_count) /* size_of([u32; ret_lane_count]) */);
+                        let size = Size::from_bytes(4 * ret_lane_count /* size_of([u32; ret_lane_count]) */);
                         alloc.get_bytes(fx, ptr, size).unwrap()
                     }
                     _ => unreachable!("{:?}", idx_const),
diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs
index 8edb883ccb5..720d2a12534 100644
--- a/compiler/rustc_codegen_cranelift/src/lib.rs
+++ b/compiler/rustc_codegen_cranelift/src/lib.rs
@@ -1,13 +1,4 @@
-#![feature(
-    rustc_private,
-    decl_macro,
-    type_alias_impl_trait,
-    associated_type_bounds,
-    never_type,
-    try_blocks,
-    box_patterns,
-    hash_drain_filter
-)]
+#![feature(rustc_private, decl_macro, never_type, hash_drain_filter)]
 #![warn(rust_2018_idioms)]
 #![warn(unused_lifetimes)]
 #![warn(unreachable_pub)]
@@ -57,6 +48,7 @@ mod base;
 mod cast;
 mod codegen_i128;
 mod common;
+mod compiler_builtins;
 mod constant;
 mod debuginfo;
 mod discriminant;
@@ -224,8 +216,10 @@ pub struct CraneliftCodegenBackend {
 
 impl CodegenBackend for CraneliftCodegenBackend {
     fn init(&self, sess: &Session) {
-        if sess.lto() != rustc_session::config::Lto::No && sess.opts.cg.embed_bitcode {
-            sess.warn("LTO is not supported. You may get a linker error.");
+        use rustc_session::config::Lto;
+        match sess.lto() {
+            Lto::No | Lto::ThinLocal => {}
+            Lto::Thin | Lto::Fat => sess.warn("LTO is not supported. You may get a linker error."),
         }
     }
 
@@ -240,9 +234,9 @@ impl CodegenBackend for CraneliftCodegenBackend {
         vec![]
     }
 
-    fn codegen_crate<'tcx>(
+    fn codegen_crate(
         &self,
-        tcx: TyCtxt<'tcx>,
+        tcx: TyCtxt<'_>,
         metadata: EncodedMetadata,
         need_metadata_module: bool,
     ) -> Box<dyn Any> {
@@ -252,9 +246,7 @@ impl CodegenBackend for CraneliftCodegenBackend {
             BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args)
                 .unwrap_or_else(|err| tcx.sess.fatal(&err))
         };
-        let res = driver::codegen_crate(tcx, metadata, need_metadata_module, config);
-
-        res
+        driver::codegen_crate(tcx, metadata, need_metadata_module, config)
     }
 
     fn join_codegen(
@@ -300,9 +292,9 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
     let mut flags_builder = settings::builder();
     flags_builder.enable("is_pic").unwrap();
     flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided
-    flags_builder
-        .set("enable_verifier", if cfg!(debug_assertions) { "true" } else { "false" })
-        .unwrap();
+    let enable_verifier =
+        cfg!(debug_assertions) || std::env::var("CG_CLIF_ENABLE_VERIFIER").is_ok();
+    flags_builder.set("enable_verifier", if enable_verifier { "true" } else { "false" }).unwrap();
 
     let tls_model = match target_triple.binary_format {
         BinaryFormat::Elf => "elf_gd",
@@ -314,18 +306,17 @@ fn build_isa(sess: &Session) -> Box<dyn isa::TargetIsa + 'static> {
 
     flags_builder.set("enable_simd", "true").unwrap();
 
+    flags_builder.set("enable_llvm_abi_extensions", "true").unwrap();
+
     use rustc_session::config::OptLevel;
     match sess.opts.optimize {
         OptLevel::No => {
             flags_builder.set("opt_level", "none").unwrap();
         }
         OptLevel::Less | OptLevel::Default => {}
-        OptLevel::Aggressive => {
+        OptLevel::Size | OptLevel::SizeMin | OptLevel::Aggressive => {
             flags_builder.set("opt_level", "speed_and_size").unwrap();
         }
-        OptLevel::Size | OptLevel::SizeMin => {
-            sess.warn("Optimizing for size is not supported. Just ignoring the request");
-        }
     }
 
     let flags = settings::Flags::new(flags_builder);
diff --git a/compiler/rustc_codegen_cranelift/src/linkage.rs b/compiler/rustc_codegen_cranelift/src/linkage.rs
index dc1e2107ce7..a564a59f725 100644
--- a/compiler/rustc_codegen_cranelift/src/linkage.rs
+++ b/compiler/rustc_codegen_cranelift/src/linkage.rs
@@ -6,8 +6,10 @@ pub(crate) fn get_clif_linkage(
     mono_item: MonoItem<'_>,
     linkage: RLinkage,
     visibility: Visibility,
+    is_compiler_builtins: bool,
 ) -> Linkage {
     match (linkage, visibility) {
+        (RLinkage::External, Visibility::Default) if is_compiler_builtins => Linkage::Hidden,
         (RLinkage::External, Visibility::Default) => Linkage::Export,
         (RLinkage::Internal, Visibility::Default) => Linkage::Local,
         (RLinkage::External, Visibility::Hidden) => Linkage::Hidden,
diff --git a/compiler/rustc_codegen_cranelift/src/main_shim.rs b/compiler/rustc_codegen_cranelift/src/main_shim.rs
index 62e551b186f..a6266f50776 100644
--- a/compiler/rustc_codegen_cranelift/src/main_shim.rs
+++ b/compiler/rustc_codegen_cranelift/src/main_shim.rs
@@ -1,3 +1,4 @@
+use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
 use rustc_hir::LangItem;
 use rustc_session::config::EntryFnType;
 
@@ -100,12 +101,8 @@ pub(crate) fn maybe_create_entry_wrapper(
             bcx.seal_all_blocks();
             bcx.finalize();
         }
-        m.define_function(
-            cmain_func_id,
-            &mut ctx,
-            &mut cranelift_codegen::binemit::NullTrapSink {},
-        )
-        .unwrap();
+        m.define_function(cmain_func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {})
+            .unwrap();
         unwind_context.add_function(cmain_func_id, &ctx, m.isa());
     }
 }
diff --git a/compiler/rustc_codegen_cranelift/src/metadata.rs b/compiler/rustc_codegen_cranelift/src/metadata.rs
index 190c4f45cca..c5189c972cd 100644
--- a/compiler/rustc_codegen_cranelift/src/metadata.rs
+++ b/compiler/rustc_codegen_cranelift/src/metadata.rs
@@ -1,11 +1,11 @@
 //! Reading and writing of the rustc metadata for rlibs and dylibs
 
-use std::convert::TryFrom;
 use std::fs::File;
+use std::ops::Deref;
 use std::path::Path;
 
 use rustc_codegen_ssa::METADATA_FILENAME;
-use rustc_data_structures::owning_ref::OwningRef;
+use rustc_data_structures::owning_ref::{OwningRef, StableAddress};
 use rustc_data_structures::rustc_erase_owner;
 use rustc_data_structures::sync::MetadataRef;
 use rustc_middle::middle::cstore::{EncodedMetadata, MetadataLoader};
@@ -17,38 +17,56 @@ use crate::backend::WriteMetadata;
 
 pub(crate) struct CraneliftMetadataLoader;
 
+struct StableMmap(memmap2::Mmap);
+
+impl Deref for StableMmap {
+    type Target = [u8];
+
+    fn deref(&self) -> &[u8] {
+        &*self.0
+    }
+}
+
+unsafe impl StableAddress for StableMmap {}
+
+fn load_metadata_with(
+    path: &Path,
+    f: impl for<'a> FnOnce(&'a [u8]) -> Result<&'a [u8], String>,
+) -> Result<MetadataRef, String> {
+    let file = File::open(path).map_err(|e| format!("{:?}", e))?;
+    let data = unsafe { memmap2::MmapOptions::new().map_copy_read_only(&file) }
+        .map_err(|e| format!("{:?}", e))?;
+    let metadata = OwningRef::new(StableMmap(data)).try_map(f)?;
+    return Ok(rustc_erase_owner!(metadata.map_owner_box()));
+}
+
 impl MetadataLoader for CraneliftMetadataLoader {
     fn get_rlib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> {
-        let mut archive = ar::Archive::new(File::open(path).map_err(|e| format!("{:?}", e))?);
-        // Iterate over all entries in the archive:
-        while let Some(entry_result) = archive.next_entry() {
-            let mut entry = entry_result.map_err(|e| format!("{:?}", e))?;
-            if entry.header().identifier() == METADATA_FILENAME.as_bytes() {
-                let mut buf = Vec::with_capacity(
-                    usize::try_from(entry.header().size())
-                        .expect("Rlib metadata file too big to load into memory."),
-                );
-                ::std::io::copy(&mut entry, &mut buf).map_err(|e| format!("{:?}", e))?;
-                let buf: OwningRef<Vec<u8>, [u8]> = OwningRef::new(buf);
-                return Ok(rustc_erase_owner!(buf.map_owner_box()));
+        load_metadata_with(path, |data| {
+            let archive = object::read::archive::ArchiveFile::parse(&*data)
+                .map_err(|e| format!("{:?}", e))?;
+
+            for entry_result in archive.members() {
+                let entry = entry_result.map_err(|e| format!("{:?}", e))?;
+                if entry.name() == METADATA_FILENAME.as_bytes() {
+                    return Ok(entry.data());
+                }
             }
-        }
 
-        Err("couldn't find metadata entry".to_string())
+            Err("couldn't find metadata entry".to_string())
+        })
     }
 
     fn get_dylib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> {
         use object::{Object, ObjectSection};
-        let file = std::fs::read(path).map_err(|e| format!("read:{:?}", e))?;
-        let file = object::File::parse(&file).map_err(|e| format!("parse: {:?}", e))?;
-        let buf = file
-            .section_by_name(".rustc")
-            .ok_or("no .rustc section")?
-            .data()
-            .map_err(|e| format!("failed to read .rustc section: {:?}", e))?
-            .to_owned();
-        let buf: OwningRef<Vec<u8>, [u8]> = OwningRef::new(buf);
-        Ok(rustc_erase_owner!(buf.map_owner_box()))
+
+        load_metadata_with(path, |data| {
+            let file = object::File::parse(&data).map_err(|e| format!("parse: {:?}", e))?;
+            file.section_by_name(".rustc")
+                .ok_or("no .rustc section")?
+                .data()
+                .map_err(|e| format!("failed to read .rustc section: {:?}", e))
+        })
     }
 }
 
diff --git a/compiler/rustc_codegen_cranelift/src/num.rs b/compiler/rustc_codegen_cranelift/src/num.rs
index da49e1c6c91..2ebf30da2d8 100644
--- a/compiler/rustc_codegen_cranelift/src/num.rs
+++ b/compiler/rustc_codegen_cranelift/src/num.rs
@@ -166,13 +166,11 @@ pub(crate) fn codegen_int_binop<'tcx>(
         BinOp::Shl => {
             let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
             let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
-            let actual_shift = clif_intcast(fx, actual_shift, types::I8, false);
             fx.bcx.ins().ishl(lhs, actual_shift)
         }
         BinOp::Shr => {
             let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
             let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
-            let actual_shift = clif_intcast(fx, actual_shift, types::I8, false);
             if signed {
                 fx.bcx.ins().sshr(lhs, actual_shift)
             } else {
@@ -387,7 +385,7 @@ pub(crate) fn codegen_ptr_binop<'tcx>(
                 let lhs = in_lhs.load_scalar(fx);
                 let rhs = in_rhs.load_scalar(fx);
 
-                return codegen_compare_bin_op(fx, bin_op, false, lhs, rhs);
+                codegen_compare_bin_op(fx, bin_op, false, lhs, rhs)
             }
             BinOp::Offset => {
                 let pointee_ty = in_lhs.layout().ty.builtin_deref(true).unwrap().ty;
@@ -396,10 +394,10 @@ pub(crate) fn codegen_ptr_binop<'tcx>(
                 let ptr_diff = fx.bcx.ins().imul_imm(offset, pointee_size as i64);
                 let base_val = base.load_scalar(fx);
                 let res = fx.bcx.ins().iadd(base_val, ptr_diff);
-                return CValue::by_val(res, base.layout());
+                CValue::by_val(res, base.layout())
             }
             _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs),
-        };
+        }
     } else {
         let (lhs_ptr, lhs_extra) = in_lhs.load_scalar_pair(fx);
         let (rhs_ptr, rhs_extra) = in_rhs.load_scalar_pair(fx);
diff --git a/compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs b/compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs
index d111f37f5e4..8bb02a3e558 100644
--- a/compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs
+++ b/compiler/rustc_codegen_cranelift/src/optimize/stack2reg.rs
@@ -181,7 +181,6 @@ impl<'a> OptimizeContext<'a> {
 
 pub(super) fn optimize_function(
     ctx: &mut Context,
-    #[cfg_attr(not(debug_assertions), allow(unused_variables))]
     clif_comments: &mut crate::pretty_clif::CommentWriter,
 ) {
     combine_stack_addr_with_load_store(&mut ctx.func);
@@ -192,8 +191,7 @@ pub(super) fn optimize_function(
 
     remove_unused_stack_addr_and_stack_load(&mut opt_ctx);
 
-    #[cfg(debug_assertions)]
-    {
+    if clif_comments.enabled() {
         for (&OrdStackSlot(stack_slot), usage) in &opt_ctx.stack_slot_usage_map {
             clif_comments.add_comment(stack_slot, format!("used by: {:?}", usage));
         }
@@ -209,25 +207,27 @@ pub(super) fn optimize_function(
         for load in users.stack_load.clone().into_iter() {
             let potential_stores = users.potential_stores_for_load(&opt_ctx.ctx, load);
 
-            #[cfg(debug_assertions)]
-            for &store in &potential_stores {
-                clif_comments.add_comment(
-                    load,
-                    format!(
-                        "Potential store -> load forwarding {} -> {} ({:?}, {:?})",
-                        opt_ctx.ctx.func.dfg.display_inst(store, None),
-                        opt_ctx.ctx.func.dfg.display_inst(load, None),
-                        spatial_overlap(&opt_ctx.ctx.func, store, load),
-                        temporal_order(&opt_ctx.ctx, store, load),
-                    ),
-                );
+            if clif_comments.enabled() {
+                for &store in &potential_stores {
+                    clif_comments.add_comment(
+                        load,
+                        format!(
+                            "Potential store -> load forwarding {} -> {} ({:?}, {:?})",
+                            opt_ctx.ctx.func.dfg.display_inst(store, None),
+                            opt_ctx.ctx.func.dfg.display_inst(load, None),
+                            spatial_overlap(&opt_ctx.ctx.func, store, load),
+                            temporal_order(&opt_ctx.ctx, store, load),
+                        ),
+                    );
+                }
             }
 
             match *potential_stores {
                 [] => {
-                    #[cfg(debug_assertions)]
-                    clif_comments
-                        .add_comment(load, "[BUG?] Reading uninitialized memory".to_string());
+                    if clif_comments.enabled() {
+                        clif_comments
+                            .add_comment(load, "[BUG?] Reading uninitialized memory".to_string());
+                    }
                 }
                 [store]
                     if spatial_overlap(&opt_ctx.ctx.func, store, load) == SpatialOverlap::Full
@@ -237,9 +237,12 @@ pub(super) fn optimize_function(
                     // Only one store could have been the origin of the value.
                     let stored_value = opt_ctx.ctx.func.dfg.inst_args(store)[0];
 
-                    #[cfg(debug_assertions)]
-                    clif_comments
-                        .add_comment(load, format!("Store to load forward {} -> {}", store, load));
+                    if clif_comments.enabled() {
+                        clif_comments.add_comment(
+                            load,
+                            format!("Store to load forward {} -> {}", store, load),
+                        );
+                    }
 
                     users.change_load_to_alias(&mut opt_ctx.ctx.func, load, stored_value);
                 }
@@ -250,33 +253,35 @@ pub(super) fn optimize_function(
         for store in users.stack_store.clone().into_iter() {
             let potential_loads = users.potential_loads_of_store(&opt_ctx.ctx, store);
 
-            #[cfg(debug_assertions)]
-            for &load in &potential_loads {
-                clif_comments.add_comment(
-                    store,
-                    format!(
-                        "Potential load from store {} <- {} ({:?}, {:?})",
-                        opt_ctx.ctx.func.dfg.display_inst(load, None),
-                        opt_ctx.ctx.func.dfg.display_inst(store, None),
-                        spatial_overlap(&opt_ctx.ctx.func, store, load),
-                        temporal_order(&opt_ctx.ctx, store, load),
-                    ),
-                );
+            if clif_comments.enabled() {
+                for &load in &potential_loads {
+                    clif_comments.add_comment(
+                        store,
+                        format!(
+                            "Potential load from store {} <- {} ({:?}, {:?})",
+                            opt_ctx.ctx.func.dfg.display_inst(load, None),
+                            opt_ctx.ctx.func.dfg.display_inst(store, None),
+                            spatial_overlap(&opt_ctx.ctx.func, store, load),
+                            temporal_order(&opt_ctx.ctx, store, load),
+                        ),
+                    );
+                }
             }
 
             if potential_loads.is_empty() {
                 // Never loaded; can safely remove all stores and the stack slot.
                 // FIXME also remove stores when there is always a next store before a load.
 
-                #[cfg(debug_assertions)]
-                clif_comments.add_comment(
-                    store,
-                    format!(
-                        "Remove dead stack store {} of {}",
-                        opt_ctx.ctx.func.dfg.display_inst(store, None),
-                        stack_slot.0
-                    ),
-                );
+                if clif_comments.enabled() {
+                    clif_comments.add_comment(
+                        store,
+                        format!(
+                            "Remove dead stack store {} of {}",
+                            opt_ctx.ctx.func.dfg.display_inst(store, None),
+                            stack_slot.0
+                        ),
+                    );
+                }
 
                 users.remove_dead_store(&mut opt_ctx.ctx.func, store);
             }
diff --git a/compiler/rustc_codegen_cranelift/src/pointer.rs b/compiler/rustc_codegen_cranelift/src/pointer.rs
index 88a78f3214d..31d827f83bf 100644
--- a/compiler/rustc_codegen_cranelift/src/pointer.rs
+++ b/compiler/rustc_codegen_cranelift/src/pointer.rs
@@ -39,8 +39,7 @@ impl Pointer {
         Pointer { base: PointerBase::Dangling(align), offset: Offset32::new(0) }
     }
 
-    #[cfg(debug_assertions)]
-    pub(crate) fn base_and_offset(self) -> (PointerBase, Offset32) {
+    pub(crate) fn debug_base_and_offset(self) -> (PointerBase, Offset32) {
         (self.base, self.offset)
     }
 
diff --git a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
index 9c91b92e515..d22ea3772ee 100644
--- a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
+++ b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
@@ -69,13 +69,15 @@ use crate::prelude::*;
 
 #[derive(Debug)]
 pub(crate) struct CommentWriter {
+    enabled: bool,
     global_comments: Vec<String>,
     entity_comments: FxHashMap<AnyEntity, String>,
 }
 
 impl CommentWriter {
     pub(crate) fn new<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self {
-        let global_comments = if cfg!(debug_assertions) {
+        let enabled = should_write_ir(tcx);
+        let global_comments = if enabled {
             vec![
                 format!("symbol {}", tcx.symbol_name(instance).name),
                 format!("instance {:?}", instance),
@@ -86,13 +88,17 @@ impl CommentWriter {
             vec![]
         };
 
-        CommentWriter { global_comments, entity_comments: FxHashMap::default() }
+        CommentWriter { enabled, global_comments, entity_comments: FxHashMap::default() }
     }
 }
 
-#[cfg(debug_assertions)]
 impl CommentWriter {
+    pub(crate) fn enabled(&self) -> bool {
+        self.enabled
+    }
+
     pub(crate) fn add_global_comment<S: Into<String>>(&mut self, comment: S) {
+        debug_assert!(self.enabled);
         self.global_comments.push(comment.into());
     }
 
@@ -101,6 +107,8 @@ impl CommentWriter {
         entity: E,
         comment: S,
     ) {
+        debug_assert!(self.enabled);
+
         use std::collections::hash_map::Entry;
         match self.entity_comments.entry(entity.into()) {
             Entry::Occupied(mut occ) => {
@@ -179,7 +187,6 @@ impl FuncWriter for &'_ CommentWriter {
     }
 }
 
-#[cfg(debug_assertions)]
 impl FunctionCx<'_, '_, '_> {
     pub(crate) fn add_global_comment<S: Into<String>>(&mut self, comment: S) {
         self.clif_comments.add_global_comment(comment);
@@ -198,8 +205,8 @@ pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool {
     tcx.sess.opts.output_types.contains_key(&OutputType::LlvmAssembly)
 }
 
-pub(crate) fn write_ir_file<'tcx>(
-    tcx: TyCtxt<'tcx>,
+pub(crate) fn write_ir_file(
+    tcx: TyCtxt<'_>,
     name: &str,
     write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>,
 ) {
@@ -217,10 +224,7 @@ pub(crate) fn write_ir_file<'tcx>(
 
     let clif_file_name = clif_output_dir.join(name);
 
-    let res: std::io::Result<()> = try {
-        let mut file = std::fs::File::create(clif_file_name)?;
-        write(&mut file)?;
-    };
+    let res = std::fs::File::create(clif_file_name).and_then(|mut file| write(&mut file));
     if let Err(err) = res {
         tcx.sess.warn(&format!("error writing ir file: {}", err));
     }
diff --git a/compiler/rustc_codegen_cranelift/src/trap.rs b/compiler/rustc_codegen_cranelift/src/trap.rs
index bb63d72addf..1ab0703e981 100644
--- a/compiler/rustc_codegen_cranelift/src/trap.rs
+++ b/compiler/rustc_codegen_cranelift/src/trap.rs
@@ -17,8 +17,7 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) {
         )
         .unwrap();
     let puts = fx.cx.module.declare_func_in_func(puts, &mut fx.bcx.func);
-    #[cfg(debug_assertions)]
-    {
+    if fx.clif_comments.enabled() {
         fx.add_comment(puts, "puts");
     }
 
diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
index cffaf79ded1..b97d3900984 100644
--- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs
+++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
@@ -2,7 +2,6 @@
 
 use crate::prelude::*;
 
-use cranelift_codegen::entity::EntityRef;
 use cranelift_codegen::ir::immediates::Offset32;
 
 fn codegen_field<'tcx>(
@@ -414,7 +413,7 @@ impl<'tcx> CPlace<'tcx> {
         self,
         fx: &mut FunctionCx<'_, '_, 'tcx>,
         from: CValue<'tcx>,
-        #[cfg_attr(not(debug_assertions), allow(unused_variables))] method: &'static str,
+        method: &'static str,
     ) {
         fn transmute_value<'tcx>(
             fx: &mut FunctionCx<'_, '_, 'tcx>,
@@ -462,8 +461,7 @@ impl<'tcx> CPlace<'tcx> {
 
         assert_eq!(self.layout().size, from.layout().size);
 
-        #[cfg(debug_assertions)]
-        {
+        if fx.clif_comments.enabled() {
             use cranelift_codegen::cursor::{Cursor, CursorPosition};
             let cur_block = match fx.bcx.cursor().position() {
                 CursorPosition::After(block) => block,
@@ -707,6 +705,19 @@ pub(crate) fn assert_assignable<'tcx>(
             }
             // dyn for<'r> Trait<'r> -> dyn Trait<'_> is allowed
         }
+        (&ty::Adt(adt_def_a, substs_a), &ty::Adt(adt_def_b, substs_b))
+            if adt_def_a.did == adt_def_b.did =>
+        {
+            let mut types_a = substs_a.types();
+            let mut types_b = substs_b.types();
+            loop {
+                match (types_a.next(), types_b.next()) {
+                    (Some(a), Some(b)) => assert_assignable(fx, a, b),
+                    (None, None) => return,
+                    (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty),
+                }
+            }
+        }
         _ => {
             assert_eq!(
                 from_ty, to_ty,