Merge remote-tracking branch 'origin/master' into mpk/add-long-error-message-for-E0311

12 files changed, 129 insertions, 283 deletions

diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs
index 7f6947e3c79..d2f2c7bf798 100644
--- a/compiler/rustc_codegen_ssa/src/back/link.rs
+++ b/compiler/rustc_codegen_ssa/src/back/link.rs
@@ -1958,7 +1958,6 @@ fn linker_with_args<'a>(
     // Upstream rust libraries are not supposed to depend on our local native
     // libraries as that would violate the structure of the DAG, in that
     // scenario they are required to link to them as well in a shared fashion.
-    // (The current implementation still doesn't prevent it though, see the FIXME below.)
     //
     // Note that upstream rust libraries may contain native dependencies as
     // well, but they also can't depend on what we just started to add to the
@@ -1979,15 +1978,16 @@ fn linker_with_args<'a>(
     // and move this option back to the top.
     cmd.add_as_needed();
 
-    // FIXME: Move this below to other native libraries
-    // (or alternatively link all native libraries after their respective crates).
-    // This change is somewhat breaking in practice due to local static libraries being linked
-    // as whole-archive (#85144), so removing whole-archive may be a pre-requisite.
+    // Local native libraries of all kinds.
+    //
+    // If `-Zlink-native-libraries=false` is set, then the assumption is that an
+    // external build system already has the native dependencies defined, and it
+    // will provide them to the linker itself.
     if sess.opts.unstable_opts.link_native_libraries {
         add_local_native_libraries(cmd, sess, codegen_results);
     }
 
-    // Upstream rust libraries and their non-bundled static libraries
+    // Upstream rust libraries and their (possibly bundled) static native libraries.
     add_upstream_rust_crates(
         cmd,
         sess,
@@ -1997,11 +1997,11 @@ fn linker_with_args<'a>(
         tmpdir,
     );
 
-    // Upstream dynamic native libraries linked with `#[link]` attributes at and `-l`
-    // command line options.
-    // If -Zlink-native-libraries=false is set, then the assumption is that an
-    // external build system already has the native dependencies defined, and it
-    // will provide them to the linker itself.
+    // Dynamic native libraries from upstream crates.
+    //
+    // FIXME: Merge this to `add_upstream_rust_crates` so that all native libraries are linked
+    // together with their respective upstream crates, and in their originally specified order.
+    // This may be slightly breaking due to our use of `--as-needed` and needs a crater run.
     if sess.opts.unstable_opts.link_native_libraries {
         add_upstream_native_libraries(cmd, sess, codegen_results);
     }
@@ -2415,7 +2415,7 @@ fn add_upstream_rust_crates<'a>(
                 // or is an rlib already included via some other dylib crate, the symbols from
                 // native libs will have already been included in that dylib.
                 //
-                // If -Zlink-native-libraries=false is set, then the assumption is that an
+                // If `-Zlink-native-libraries=false` is set, then the assumption is that an
                 // external build system already has the native dependencies defined, and it
                 // will provide them to the linker itself.
                 if sess.opts.unstable_opts.link_native_libraries {
diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs
index 0302c28815a..b92e146bee2 100644
--- a/compiler/rustc_codegen_ssa/src/back/metadata.rs
+++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs
@@ -187,12 +187,12 @@ pub enum MetadataPosition {
     Last,
 }
 
-// For rlibs we "pack" rustc metadata into a dummy object file. When rustc
-// creates a dylib crate type it will pass `--whole-archive` (or the
-// platform equivalent) to include all object files from an rlib into the
-// final dylib itself. This causes linkers to iterate and try to include all
-// files located in an archive, so if metadata is stored in an archive then
-// it needs to be of a form that the linker will be able to process.
+// For rlibs we "pack" rustc metadata into a dummy object file.
+//
+// Historically it was needed because rustc linked rlibs as whole-archive in some cases.
+// In that case linkers try to include all files located in an archive, so if metadata is stored
+// in an archive then it needs to be of a form that the linker is able to process.
+// Now it's not clear whether metadata still needs to be wrapped into an object file or not.
 //
 // Note, though, that we don't actually want this metadata to show up in any
 // final output of the compiler. Instead this is purely for rustc's own
diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
index e6b6055759d..32b340832ce 100644
--- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
+++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
@@ -103,18 +103,14 @@ fn reachable_non_generics_provider(tcx: TyCtxt<'_>, cnum: CrateNum) -> DefIdMap<
             }
         })
         .map(|def_id| {
-            let (export_level, used) = if special_runtime_crate {
-                let name = tcx.symbol_name(Instance::mono(tcx, def_id.to_def_id())).name;
-                // We won't link right if these symbols are stripped during LTO.
-                let used = match name {
-                    "rust_eh_personality"
-                    | "rust_eh_register_frames"
-                    | "rust_eh_unregister_frames" => true,
-                    _ => false,
-                };
-                (SymbolExportLevel::Rust, used)
+            // We won't link right if this symbol is stripped during LTO.
+            let name = tcx.symbol_name(Instance::mono(tcx, def_id.to_def_id())).name;
+            let used = name == "rust_eh_personality";
+
+            let export_level = if special_runtime_crate {
+                SymbolExportLevel::Rust
             } else {
-                (symbol_export_level(tcx, def_id.to_def_id()), false)
+                symbol_export_level(tcx, def_id.to_def_id())
             };
             let codegen_attrs = tcx.codegen_fn_attrs(def_id.to_def_id());
             debug!(
diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs
index 2930d09d71f..68f3b19b715 100644
--- a/compiler/rustc_codegen_ssa/src/back/write.rs
+++ b/compiler/rustc_codegen_ssa/src/back/write.rs
@@ -1892,7 +1892,7 @@ impl<B: ExtraBackendMethods> OngoingCodegen<B> {
             }
         });
 
-        sess.cgu_reuse_tracker.check_expected_reuse(sess.diagnostic());
+        sess.cgu_reuse_tracker.check_expected_reuse(sess);
 
         sess.abort_if_errors();
 
diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs
index a840b270974..4c6be3f9108 100644
--- a/compiler/rustc_codegen_ssa/src/base.rs
+++ b/compiler/rustc_codegen_ssa/src/base.rs
@@ -151,6 +151,7 @@ pub fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
             let old_info =
                 old_info.expect("unsized_info: missing old info for trait upcasting coercion");
             if data_a.principal_def_id() == data_b.principal_def_id() {
+                // A NOP cast that doesn't actually change anything, should be allowed even with invalid vtables.
                 return old_info;
             }
 
diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs
index 1802eedf193..0faf51b062b 100644
--- a/compiler/rustc_codegen_ssa/src/lib.rs
+++ b/compiler/rustc_codegen_ssa/src/lib.rs
@@ -168,6 +168,13 @@ pub struct CodegenResults {
     pub crate_info: CrateInfo,
 }
 
+pub enum CodegenErrors<'a> {
+    WrongFileType,
+    EmptyVersionNumber,
+    EncodingVersionMismatch { version_array: String, rlink_version: u32 },
+    RustcVersionMismatch { rustc_version: String, current_version: &'a str },
+}
+
 pub fn provide(providers: &mut Providers) {
     crate::back::symbol_export::provide(providers);
     crate::base::provide(providers);
@@ -212,30 +219,34 @@ impl CodegenResults {
         encoder.finish()
     }
 
-    pub fn deserialize_rlink(data: Vec<u8>) -> Result<Self, String> {
+    pub fn deserialize_rlink<'a>(data: Vec<u8>) -> Result<Self, CodegenErrors<'a>> {
         // The Decodable machinery is not used here because it panics if the input data is invalid
         // and because its internal representation may change.
         if !data.starts_with(RLINK_MAGIC) {
-            return Err("The input does not look like a .rlink file".to_string());
+            return Err(CodegenErrors::WrongFileType);
         }
         let data = &data[RLINK_MAGIC.len()..];
         if data.len() < 4 {
-            return Err("The input does not contain version number".to_string());
+            return Err(CodegenErrors::EmptyVersionNumber);
         }
 
         let mut version_array: [u8; 4] = Default::default();
         version_array.copy_from_slice(&data[..4]);
         if u32::from_be_bytes(version_array) != RLINK_VERSION {
-            return Err(".rlink file was produced with encoding version {version_array}, but the current version is {RLINK_VERSION}".to_string());
+            return Err(CodegenErrors::EncodingVersionMismatch {
+                version_array: String::from_utf8_lossy(&version_array).to_string(),
+                rlink_version: RLINK_VERSION,
+            });
         }
 
         let mut decoder = MemDecoder::new(&data[4..], 0);
         let rustc_version = decoder.read_str();
         let current_version = RUSTC_VERSION.unwrap();
         if rustc_version != current_version {
-            return Err(format!(
-                ".rlink file was produced by rustc version {rustc_version}, but the current version is {current_version}."
-            ));
+            return Err(CodegenErrors::RustcVersionMismatch {
+                rustc_version: rustc_version.to_string(),
+                current_version,
+            });
         }
 
         let codegen_results = CodegenResults::decode(&mut decoder);
diff --git a/compiler/rustc_codegen_ssa/src/meth.rs b/compiler/rustc_codegen_ssa/src/meth.rs
index 27d791d90a5..f8e982b7751 100644
--- a/compiler/rustc_codegen_ssa/src/meth.rs
+++ b/compiler/rustc_codegen_ssa/src/meth.rs
@@ -1,6 +1,6 @@
 use crate::traits::*;
 
-use rustc_middle::ty::{self, subst::GenericArgKind, ExistentialPredicate, Ty, TyCtxt};
+use rustc_middle::ty::{self, subst::GenericArgKind, Ty};
 use rustc_session::config::Lto;
 use rustc_symbol_mangling::typeid_for_trait_ref;
 use rustc_target::abi::call::FnAbi;
@@ -29,7 +29,7 @@ impl<'a, 'tcx> VirtualIndex {
             && bx.cx().sess().lto() == Lto::Fat
         {
             let typeid =
-                bx.typeid_metadata(typeid_for_trait_ref(bx.tcx(), get_trait_ref(bx.tcx(), ty)));
+                bx.typeid_metadata(typeid_for_trait_ref(bx.tcx(), expect_dyn_trait_in_self(ty)));
             let vtable_byte_offset = self.0 * bx.data_layout().pointer_size.bytes();
             let type_checked_load = bx.type_checked_load(llvtable, vtable_byte_offset, typeid);
             let func = bx.extract_value(type_checked_load, 0);
@@ -64,17 +64,13 @@ impl<'a, 'tcx> VirtualIndex {
     }
 }
 
-fn get_trait_ref<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ty::PolyExistentialTraitRef<'tcx> {
+/// This takes a valid `self` receiver type and extracts the principal trait
+/// ref of the type.
+fn expect_dyn_trait_in_self<'tcx>(ty: Ty<'tcx>) -> ty::PolyExistentialTraitRef<'tcx> {
     for arg in ty.peel_refs().walk() {
         if let GenericArgKind::Type(ty) = arg.unpack() {
-            if let ty::Dynamic(trait_refs, _) = ty.kind() {
-                return trait_refs[0].map_bound(|trait_ref| match trait_ref {
-                    ExistentialPredicate::Trait(tr) => tr,
-                    ExistentialPredicate::Projection(proj) => proj.trait_ref(tcx),
-                    ExistentialPredicate::AutoTrait(_) => {
-                        bug!("auto traits don't have functions")
-                    }
-                });
+            if let ty::Dynamic(data, _) = ty.kind() {
+                return data.principal().expect("expected principal trait object");
             }
         }
     }
diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
index 24da48ead63..c7617d2e464 100644
--- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
@@ -266,7 +266,7 @@ pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKi
         result: &mut IndexVec<mir::BasicBlock, CleanupKind>,
         mir: &mir::Body<'tcx>,
     ) {
-        for (bb, data) in mir.basic_blocks().iter_enumerated() {
+        for (bb, data) in mir.basic_blocks.iter_enumerated() {
             match data.terminator().kind {
                 TerminatorKind::Goto { .. }
                 | TerminatorKind::Resume
@@ -296,7 +296,7 @@ pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKi
     }
 
     fn propagate<'tcx>(result: &mut IndexVec<mir::BasicBlock, CleanupKind>, mir: &mir::Body<'tcx>) {
-        let mut funclet_succs = IndexVec::from_elem(None, mir.basic_blocks());
+        let mut funclet_succs = IndexVec::from_elem(None, &mir.basic_blocks);
 
         let mut set_successor = |funclet: mir::BasicBlock, succ| match funclet_succs[funclet] {
             ref mut s @ None => {
@@ -359,7 +359,7 @@ pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKi
         }
     }
 
-    let mut result = IndexVec::from_elem(CleanupKind::NotCleanup, mir.basic_blocks());
+    let mut result = IndexVec::from_elem(CleanupKind::NotCleanup, &mir.basic_blocks);
 
     discover_masters(&mut result, mir);
     propagate(&mut result, mir);
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index 3eee58d9d1c..5c67d3b6431 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -21,7 +21,7 @@ use rustc_middle::ty::{self, Instance, Ty, TypeVisitable};
 use rustc_span::source_map::Span;
 use rustc_span::{sym, Symbol};
 use rustc_symbol_mangling::typeid::typeid_for_fnabi;
-use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode};
+use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode, Reg};
 use rustc_target::abi::{self, HasDataLayout, WrappingRange};
 use rustc_target::spec::abi::Abi;
 
@@ -324,7 +324,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             bx.unreachable();
             return;
         }
-        let llval = match self.fn_abi.ret.mode {
+        let llval = match &self.fn_abi.ret.mode {
             PassMode::Ignore | PassMode::Indirect { .. } => {
                 bx.ret_void();
                 return;
@@ -339,7 +339,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
             }
 
-            PassMode::Cast(cast_ty) => {
+            PassMode::Cast(cast_ty, _) => {
                 let op = match self.locals[mir::RETURN_PLACE] {
                     LocalRef::Operand(Some(op)) => op,
                     LocalRef::Operand(None) => bug!("use of return before def"),
@@ -360,7 +360,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         llval
                     }
                 };
-                let ty = bx.cast_backend_type(&cast_ty);
+                let ty = bx.cast_backend_type(cast_ty);
                 let addr = bx.pointercast(llslot, bx.type_ptr_to(ty));
                 bx.load(ty, addr, self.fn_abi.ret.layout.align.abi)
             }
@@ -798,58 +798,55 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             let mut op = self.codegen_operand(&mut bx, arg);
 
             if let (0, Some(ty::InstanceDef::Virtual(_, idx))) = (i, def) {
-                if let Pair(..) = op.val {
-                    // In the case of Rc<Self>, we need to explicitly pass a
-                    // *mut RcBox<Self> with a Scalar (not ScalarPair) ABI. This is a hack
-                    // that is understood elsewhere in the compiler as a method on
-                    // `dyn Trait`.
-                    // To get a `*mut RcBox<Self>`, we just keep unwrapping newtypes until
-                    // we get a value of a built-in pointer type
-                    'descend_newtypes: while !op.layout.ty.is_unsafe_ptr()
-                        && !op.layout.ty.is_region_ptr()
-                    {
-                        for i in 0..op.layout.fields.count() {
-                            let field = op.extract_field(&mut bx, i);
-                            if !field.layout.is_zst() {
-                                // we found the one non-zero-sized field that is allowed
-                                // now find *its* non-zero-sized field, or stop if it's a
-                                // pointer
-                                op = field;
-                                continue 'descend_newtypes;
+                match op.val {
+                    Pair(data_ptr, meta) => {
+                        // In the case of Rc<Self>, we need to explicitly pass a
+                        // *mut RcBox<Self> with a Scalar (not ScalarPair) ABI. This is a hack
+                        // that is understood elsewhere in the compiler as a method on
+                        // `dyn Trait`.
+                        // To get a `*mut RcBox<Self>`, we just keep unwrapping newtypes until
+                        // we get a value of a built-in pointer type
+                        'descend_newtypes: while !op.layout.ty.is_unsafe_ptr()
+                            && !op.layout.ty.is_region_ptr()
+                        {
+                            for i in 0..op.layout.fields.count() {
+                                let field = op.extract_field(&mut bx, i);
+                                if !field.layout.is_zst() {
+                                    // we found the one non-zero-sized field that is allowed
+                                    // now find *its* non-zero-sized field, or stop if it's a
+                                    // pointer
+                                    op = field;
+                                    continue 'descend_newtypes;
+                                }
                             }
+
+                            span_bug!(span, "receiver has no non-zero-sized fields {:?}", op);
                         }
 
-                        span_bug!(span, "receiver has no non-zero-sized fields {:?}", op);
+                        // now that we have `*dyn Trait` or `&dyn Trait`, split it up into its
+                        // data pointer and vtable. Look up the method in the vtable, and pass
+                        // the data pointer as the first argument
+                        llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(
+                            &mut bx,
+                            meta,
+                            op.layout.ty,
+                            &fn_abi,
+                        ));
+                        llargs.push(data_ptr);
+                        continue 'make_args;
                     }
-
-                    // now that we have `*dyn Trait` or `&dyn Trait`, split it up into its
-                    // data pointer and vtable. Look up the method in the vtable, and pass
-                    // the data pointer as the first argument
-                    match op.val {
-                        Pair(data_ptr, meta) => {
-                            llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(
-                                &mut bx,
-                                meta,
-                                op.layout.ty,
-                                &fn_abi,
-                            ));
-                            llargs.push(data_ptr);
-                            continue 'make_args;
-                        }
-                        other => bug!("expected a Pair, got {:?}", other),
+                    Ref(data_ptr, Some(meta), _) => {
+                        // by-value dynamic dispatch
+                        llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(
+                            &mut bx,
+                            meta,
+                            op.layout.ty,
+                            &fn_abi,
+                        ));
+                        llargs.push(data_ptr);
+                        continue;
                     }
-                } else if let Ref(data_ptr, Some(meta), _) = op.val {
-                    // by-value dynamic dispatch
-                    llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(
-                        &mut bx,
-                        meta,
-                        op.layout.ty,
-                        &fn_abi,
-                    ));
-                    llargs.push(data_ptr);
-                    continue;
-                } else {
-                    span_bug!(span, "can't codegen a virtual call on {:?}", op);
+                    _ => span_bug!(span, "can't codegen a virtual call on {:?}", op),
                 }
             }
 
@@ -1161,39 +1158,35 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         llargs: &mut Vec<Bx::Value>,
         arg: &ArgAbi<'tcx, Ty<'tcx>>,
     ) {
-        // Fill padding with undef value, where applicable.
-        if let Some(ty) = arg.pad {
-            llargs.push(bx.const_undef(bx.reg_backend_type(&ty)))
-        }
-
-        if arg.is_ignore() {
-            return;
-        }
-
-        if let PassMode::Pair(..) = arg.mode {
-            match op.val {
+        match arg.mode {
+            PassMode::Ignore => return,
+            PassMode::Cast(_, true) => {
+                // Fill padding with undef value, where applicable.
+                llargs.push(bx.const_undef(bx.reg_backend_type(&Reg::i32())));
+            }
+            PassMode::Pair(..) => match op.val {
                 Pair(a, b) => {
                     llargs.push(a);
                     llargs.push(b);
                     return;
                 }
                 _ => bug!("codegen_argument: {:?} invalid for pair argument", op),
-            }
-        } else if arg.is_unsized_indirect() {
-            match op.val {
+            },
+            PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => match op.val {
                 Ref(a, Some(b), _) => {
                     llargs.push(a);
                     llargs.push(b);
                     return;
                 }
                 _ => bug!("codegen_argument: {:?} invalid for unsized indirect argument", op),
-            }
+            },
+            _ => {}
         }
 
         // Force by-ref if we have to load through a cast pointer.
         let (mut llval, align, by_ref) = match op.val {
             Immediate(_) | Pair(..) => match arg.mode {
-                PassMode::Indirect { .. } | PassMode::Cast(_) => {
+                PassMode::Indirect { .. } | PassMode::Cast(..) => {
                     let scratch = PlaceRef::alloca(bx, arg.layout);
                     op.val.store(bx, scratch);
                     (scratch.llval, scratch.align, true)
@@ -1225,8 +1218,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
         if by_ref && !arg.is_indirect() {
             // Have to load the argument, maybe while casting it.
-            if let PassMode::Cast(ty) = arg.mode {
-                let llty = bx.cast_backend_type(&ty);
+            if let PassMode::Cast(ty, _) = &arg.mode {
+                let llty = bx.cast_backend_type(ty);
                 let addr = bx.pointercast(llval, bx.type_ptr_to(llty));
                 llval = bx.load(llty, addr, align.min(arg.layout.align.abi));
             } else {
@@ -1625,7 +1618,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             }
             DirectOperand(index) => {
                 // If there is a cast, we have to store and reload.
-                let op = if let PassMode::Cast(_) = ret_abi.mode {
+                let op = if let PassMode::Cast(..) = ret_abi.mode {
                     let tmp = PlaceRef::alloca(bx, ret_abi.layout);
                     tmp.storage_live(bx);
                     bx.store_arg(&ret_abi, llval, tmp);
diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
index 94ac71a4dd2..16aad07194d 100644
--- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
@@ -597,8 +597,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         };
 
         if !fn_abi.ret.is_ignore() {
-            if let PassMode::Cast(ty) = fn_abi.ret.mode {
-                let ptr_llty = bx.type_ptr_to(bx.cast_backend_type(&ty));
+            if let PassMode::Cast(ty, _) = &fn_abi.ret.mode {
+                let ptr_llty = bx.type_ptr_to(bx.cast_backend_type(ty));
                 let ptr = bx.pointercast(result.llval, ptr_llty);
                 bx.store(llval, ptr, result.align);
             } else {
diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs
index 8ee375fa9e3..d6bbcd99234 100644
--- a/compiler/rustc_codegen_ssa/src/mir/mod.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs
@@ -150,13 +150,13 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     let start_llbb = Bx::append_block(cx, llfn, "start");
     let mut bx = Bx::build(cx, start_llbb);
 
-    if mir.basic_blocks().iter().any(|bb| bb.is_cleanup) {
+    if mir.basic_blocks.iter().any(|bb| bb.is_cleanup) {
         bx.set_personality_fn(cx.eh_personality());
     }
 
     let cleanup_kinds = analyze::cleanup_kinds(&mir);
     let cached_llbbs: IndexVec<mir::BasicBlock, Option<Bx::BasicBlock>> = mir
-        .basic_blocks()
+        .basic_blocks
         .indices()
         .map(|bb| if bb == mir::START_BLOCK { Some(start_llbb) } else { None })
         .collect();
@@ -172,8 +172,8 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         unreachable_block: None,
         double_unwind_guard: None,
         cleanup_kinds,
-        landing_pads: IndexVec::from_elem(None, mir.basic_blocks()),
-        funclets: IndexVec::from_fn_n(|_| None, mir.basic_blocks().len()),
+        landing_pads: IndexVec::from_elem(None, &mir.basic_blocks),
+        funclets: IndexVec::from_fn_n(|_| None, mir.basic_blocks.len()),
         locals: IndexVec::new(),
         debug_context,
         per_local_var_debug_info: None,
@@ -283,7 +283,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                 for i in 0..tupled_arg_tys.len() {
                     let arg = &fx.fn_abi.args[idx];
                     idx += 1;
-                    if arg.pad.is_some() {
+                    if let PassMode::Cast(_, true) = arg.mode {
                         llarg_idx += 1;
                     }
                     let pr_field = place.project_field(bx, i);
@@ -309,7 +309,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 
             let arg = &fx.fn_abi.args[idx];
             idx += 1;
-            if arg.pad.is_some() {
+            if let PassMode::Cast(_, true) = arg.mode {
                 llarg_idx += 1;
             }
 
diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs
index 9f49749bb41..10cf8948b5a 100644
--- a/compiler/rustc_codegen_ssa/src/traits/builder.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs
@@ -1,6 +1,5 @@
 use super::abi::AbiBuilderMethods;
 use super::asm::AsmBuilderMethods;
-use super::consts::ConstMethods;
 use super::coverageinfo::CoverageInfoBuilderMethods;
 use super::debuginfo::DebugInfoBuilderMethods;
 use super::intrinsic::IntrinsicCallMethods;
@@ -15,7 +14,6 @@ use crate::mir::operand::OperandRef;
 use crate::mir::place::PlaceRef;
 use crate::MemFlags;
 
-use rustc_apfloat::{ieee, Float, Round, Status};
 use rustc_middle::ty::layout::{HasParamEnv, TyAndLayout};
 use rustc_middle::ty::Ty;
 use rustc_span::Span;
@@ -188,8 +186,8 @@ pub trait BuilderMethods<'a, 'tcx>:
 
     fn trunc(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
     fn sext(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
-    fn fptoui_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Option<Self::Value>;
-    fn fptosi_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Option<Self::Value>;
+    fn fptoui_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
+    fn fptosi_sat(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
     fn fptoui(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
     fn fptosi(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
     fn uitofp(&mut self, val: Self::Value, dest_ty: Self::Type) -> Self::Value;
@@ -223,156 +221,7 @@ pub trait BuilderMethods<'a, 'tcx>:
             return if signed { self.fptosi(x, dest_ty) } else { self.fptoui(x, dest_ty) };
         }
 
-        let try_sat_result =
-            if signed { self.fptosi_sat(x, dest_ty) } else { self.fptoui_sat(x, dest_ty) };
-        if let Some(try_sat_result) = try_sat_result {
-            return try_sat_result;
-        }
-
-        let int_width = self.cx().int_width(int_ty);
-        let float_width = self.cx().float_width(float_ty);
-        // LLVM's fpto[su]i returns undef when the input x is infinite, NaN, or does not fit into the
-        // destination integer type after rounding towards zero. This `undef` value can cause UB in
-        // safe code (see issue #10184), so we implement a saturating conversion on top of it:
-        // Semantically, the mathematical value of the input is rounded towards zero to the next
-        // mathematical integer, and then the result is clamped into the range of the destination
-        // integer type. Positive and negative infinity are mapped to the maximum and minimum value of
-        // the destination integer type. NaN is mapped to 0.
-        //
-        // Define f_min and f_max as the largest and smallest (finite) floats that are exactly equal to
-        // a value representable in int_ty.
-        // They are exactly equal to int_ty::{MIN,MAX} if float_ty has enough significand bits.
-        // Otherwise, int_ty::MAX must be rounded towards zero, as it is one less than a power of two.
-        // int_ty::MIN, however, is either zero or a negative power of two and is thus exactly
-        // representable. Note that this only works if float_ty's exponent range is sufficiently large.
-        // f16 or 256 bit integers would break this property. Right now the smallest float type is f32
-        // with exponents ranging up to 127, which is barely enough for i128::MIN = -2^127.
-        // On the other hand, f_max works even if int_ty::MAX is greater than float_ty::MAX. Because
-        // we're rounding towards zero, we just get float_ty::MAX (which is always an integer).
-        // This already happens today with u128::MAX = 2^128 - 1 > f32::MAX.
-        let int_max = |signed: bool, int_width: u64| -> u128 {
-            let shift_amount = 128 - int_width;
-            if signed { i128::MAX as u128 >> shift_amount } else { u128::MAX >> shift_amount }
-        };
-        let int_min = |signed: bool, int_width: u64| -> i128 {
-            if signed { i128::MIN >> (128 - int_width) } else { 0 }
-        };
-
-        let compute_clamp_bounds_single = |signed: bool, int_width: u64| -> (u128, u128) {
-            let rounded_min =
-                ieee::Single::from_i128_r(int_min(signed, int_width), Round::TowardZero);
-            assert_eq!(rounded_min.status, Status::OK);
-            let rounded_max =
-                ieee::Single::from_u128_r(int_max(signed, int_width), Round::TowardZero);
-            assert!(rounded_max.value.is_finite());
-            (rounded_min.value.to_bits(), rounded_max.value.to_bits())
-        };
-        let compute_clamp_bounds_double = |signed: bool, int_width: u64| -> (u128, u128) {
-            let rounded_min =
-                ieee::Double::from_i128_r(int_min(signed, int_width), Round::TowardZero);
-            assert_eq!(rounded_min.status, Status::OK);
-            let rounded_max =
-                ieee::Double::from_u128_r(int_max(signed, int_width), Round::TowardZero);
-            assert!(rounded_max.value.is_finite());
-            (rounded_min.value.to_bits(), rounded_max.value.to_bits())
-        };
-        // To implement saturation, we perform the following steps:
-        //
-        // 1. Cast x to an integer with fpto[su]i. This may result in undef.
-        // 2. Compare x to f_min and f_max, and use the comparison results to select:
-        //  a) int_ty::MIN if x < f_min or x is NaN
-        //  b) int_ty::MAX if x > f_max
-        //  c) the result of fpto[su]i otherwise
-        // 3. If x is NaN, return 0.0, otherwise return the result of step 2.
-        //
-        // This avoids resulting undef because values in range [f_min, f_max] by definition fit into the
-        // destination type. It creates an undef temporary, but *producing* undef is not UB. Our use of
-        // undef does not introduce any non-determinism either.
-        // More importantly, the above procedure correctly implements saturating conversion.
-        // Proof (sketch):
-        // If x is NaN, 0 is returned by definition.
-        // Otherwise, x is finite or infinite and thus can be compared with f_min and f_max.
-        // This yields three cases to consider:
-        // (1) if x in [f_min, f_max], the result of fpto[su]i is returned, which agrees with
-        //     saturating conversion for inputs in that range.
-        // (2) if x > f_max, then x is larger than int_ty::MAX. This holds even if f_max is rounded
-        //     (i.e., if f_max < int_ty::MAX) because in those cases, nextUp(f_max) is already larger
-        //     than int_ty::MAX. Because x is larger than int_ty::MAX, the return value of int_ty::MAX
-        //     is correct.
-        // (3) if x < f_min, then x is smaller than int_ty::MIN. As shown earlier, f_min exactly equals
-        //     int_ty::MIN and therefore the return value of int_ty::MIN is correct.
-        // QED.
-
-        let float_bits_to_llval = |bx: &mut Self, bits| {
-            let bits_llval = match float_width {
-                32 => bx.cx().const_u32(bits as u32),
-                64 => bx.cx().const_u64(bits as u64),
-                n => bug!("unsupported float width {}", n),
-            };
-            bx.bitcast(bits_llval, float_ty)
-        };
-        let (f_min, f_max) = match float_width {
-            32 => compute_clamp_bounds_single(signed, int_width),
-            64 => compute_clamp_bounds_double(signed, int_width),
-            n => bug!("unsupported float width {}", n),
-        };
-        let f_min = float_bits_to_llval(self, f_min);
-        let f_max = float_bits_to_llval(self, f_max);
-        let int_max = self.cx().const_uint_big(int_ty, int_max(signed, int_width));
-        let int_min = self.cx().const_uint_big(int_ty, int_min(signed, int_width) as u128);
-        let zero = self.cx().const_uint(int_ty, 0);
-
-        // If we're working with vectors, constants must be "splatted": the constant is duplicated
-        // into each lane of the vector.  The algorithm stays the same, we are just using the
-        // same constant across all lanes.
-        let maybe_splat = |bx: &mut Self, val| {
-            if bx.cx().type_kind(dest_ty) == TypeKind::Vector {
-                bx.vector_splat(bx.vector_length(dest_ty), val)
-            } else {
-                val
-            }
-        };
-        let f_min = maybe_splat(self, f_min);
-        let f_max = maybe_splat(self, f_max);
-        let int_max = maybe_splat(self, int_max);
-        let int_min = maybe_splat(self, int_min);
-        let zero = maybe_splat(self, zero);
-
-        // Step 1 ...
-        let fptosui_result = if signed { self.fptosi(x, dest_ty) } else { self.fptoui(x, dest_ty) };
-        let less_or_nan = self.fcmp(RealPredicate::RealULT, x, f_min);
-        let greater = self.fcmp(RealPredicate::RealOGT, x, f_max);
-
-        // Step 2: We use two comparisons and two selects, with %s1 being the
-        // result:
-        //     %less_or_nan = fcmp ult %x, %f_min
-        //     %greater = fcmp olt %x, %f_max
-        //     %s0 = select %less_or_nan, int_ty::MIN, %fptosi_result
-        //     %s1 = select %greater, int_ty::MAX, %s0
-        // Note that %less_or_nan uses an *unordered* comparison. This
-        // comparison is true if the operands are not comparable (i.e., if x is
-        // NaN). The unordered comparison ensures that s1 becomes int_ty::MIN if
-        // x is NaN.
-        //
-        // Performance note: Unordered comparison can be lowered to a "flipped"
-        // comparison and a negation, and the negation can be merged into the
-        // select. Therefore, it not necessarily any more expensive than an
-        // ordered ("normal") comparison. Whether these optimizations will be
-        // performed is ultimately up to the backend, but at least x86 does
-        // perform them.
-        let s0 = self.select(less_or_nan, int_min, fptosui_result);
-        let s1 = self.select(greater, int_max, s0);
-
-        // Step 3: NaN replacement.
-        // For unsigned types, the above step already yielded int_ty::MIN == 0 if x is NaN.
-        // Therefore we only need to execute this step for signed integer types.
-        if signed {
-            // LLVM has no isNaN predicate, so we use (x == x) instead
-            let cmp = self.fcmp(RealPredicate::RealOEQ, x, x);
-            self.select(cmp, s1, zero)
-        } else {
-            s1
-        }
+        if signed { self.fptosi_sat(x, dest_ty) } else { self.fptoui_sat(x, dest_ty) }
     }
 
     fn icmp(&mut self, op: IntPredicate, lhs: Self::Value, rhs: Self::Value) -> Self::Value;