Merge pull request #19331 from lnicola/sync-from-rust

minor: Sync from downstream

17 files changed, 635 insertions, 288 deletions

diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs
index 5054ae561c0..7d9971c021d 100644
--- a/compiler/rustc_codegen_ssa/src/back/link.rs
+++ b/compiler/rustc_codegen_ssa/src/back/link.rs
@@ -1,3 +1,5 @@
+mod raw_dylib;
+
 use std::collections::BTreeSet;
 use std::ffi::OsString;
 use std::fs::{File, OpenOptions, read};
@@ -12,7 +14,7 @@ use itertools::Itertools;
 use regex::Regex;
 use rustc_arena::TypedArena;
 use rustc_ast::CRATE_NODE_ID;
-use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
+use rustc_data_structures::fx::FxIndexSet;
 use rustc_data_structures::memmap::Mmap;
 use rustc_data_structures::temp_dir::MaybeTempDir;
 use rustc_errors::{DiagCtxtHandle, LintDiagnostic};
@@ -30,7 +32,6 @@ use rustc_session::config::{
     self, CFGuard, CrateType, DebugInfo, LinkerFeaturesCli, OutFileName, OutputFilenames,
     OutputType, PrintKind, SplitDwarfKind, Strip,
 };
-use rustc_session::cstore::DllImport;
 use rustc_session::lint::builtin::LINKER_MESSAGES;
 use rustc_session::output::{check_file_is_writeable, invalid_output_for_target, out_filename};
 use rustc_session::search_paths::PathKind;
@@ -41,22 +42,21 @@ use rustc_session::{Session, filesearch};
 use rustc_span::Symbol;
 use rustc_target::spec::crt_objects::CrtObjects;
 use rustc_target::spec::{
-    Cc, LinkOutputKind, LinkSelfContainedComponents, LinkSelfContainedDefault, LinkerFeatures,
-    LinkerFlavor, LinkerFlavorCli, Lld, PanicStrategy, RelocModel, RelroLevel, SanitizerSet,
-    SplitDebuginfo,
+    BinaryFormat, Cc, LinkOutputKind, LinkSelfContainedComponents, LinkSelfContainedDefault,
+    LinkerFeatures, LinkerFlavor, LinkerFlavorCli, Lld, PanicStrategy, RelocModel, RelroLevel,
+    SanitizerSet, SplitDebuginfo,
 };
 use tempfile::Builder as TempFileBuilder;
 use tracing::{debug, info, warn};
 
-use super::archive::{ArchiveBuilder, ArchiveBuilderBuilder, ImportLibraryItem};
+use super::archive::{ArchiveBuilder, ArchiveBuilderBuilder};
 use super::command::Command;
 use super::linker::{self, Linker};
 use super::metadata::{MetadataPosition, create_wrapper_file};
 use super::rpath::{self, RPathConfig};
 use super::{apple, versioned_llvm_target};
 use crate::{
-    CodegenResults, CompiledModule, CrateInfo, NativeLib, common, errors,
-    looks_like_rust_object_file,
+    CodegenResults, CompiledModule, CrateInfo, NativeLib, errors, looks_like_rust_object_file,
 };
 
 pub fn ensure_removed(dcx: DiagCtxtHandle<'_>, path: &Path) {
@@ -376,16 +376,22 @@ fn link_rlib<'a>(
         }
     }
 
-    for output_path in create_dll_import_libs(
-        sess,
-        archive_builder_builder,
-        codegen_results.crate_info.used_libraries.iter(),
-        tmpdir.as_ref(),
-        true,
-    ) {
-        ab.add_archive(&output_path, Box::new(|_| false)).unwrap_or_else(|error| {
-            sess.dcx().emit_fatal(errors::AddNativeLibrary { library_path: output_path, error });
-        });
+    // On Windows, we add the raw-dylib import libraries to the rlibs already.
+    // But on ELF, this is not possible, as a shared object cannot be a member of a static library.
+    // Instead, we add all raw-dylibs to the final link on ELF.
+    if sess.target.is_like_windows {
+        for output_path in raw_dylib::create_raw_dylib_dll_import_libs(
+            sess,
+            archive_builder_builder,
+            codegen_results.crate_info.used_libraries.iter(),
+            tmpdir.as_ref(),
+            true,
+        ) {
+            ab.add_archive(&output_path, Box::new(|_| false)).unwrap_or_else(|error| {
+                sess.dcx()
+                    .emit_fatal(errors::AddNativeLibrary { library_path: output_path, error });
+            });
+        }
     }
 
     if let Some(trailing_metadata) = trailing_metadata {
@@ -426,108 +432,6 @@ fn link_rlib<'a>(
     ab
 }
 
-/// Extract all symbols defined in raw-dylib libraries, collated by library name.
-///
-/// If we have multiple extern blocks that specify symbols defined in the same raw-dylib library,
-/// then the CodegenResults value contains one NativeLib instance for each block. However, the
-/// linker appears to expect only a single import library for each library used, so we need to
-/// collate the symbols together by library name before generating the import libraries.
-fn collate_raw_dylibs<'a>(
-    sess: &Session,
-    used_libraries: impl IntoIterator<Item = &'a NativeLib>,
-) -> Vec<(String, Vec<DllImport>)> {
-    // Use index maps to preserve original order of imports and libraries.
-    let mut dylib_table = FxIndexMap::<String, FxIndexMap<Symbol, &DllImport>>::default();
-
-    for lib in used_libraries {
-        if lib.kind == NativeLibKind::RawDylib {
-            let ext = if lib.verbatim { "" } else { ".dll" };
-            let name = format!("{}{}", lib.name, ext);
-            let imports = dylib_table.entry(name.clone()).or_default();
-            for import in &lib.dll_imports {
-                if let Some(old_import) = imports.insert(import.name, import) {
-                    // FIXME: when we add support for ordinals, figure out if we need to do anything
-                    // if we have two DllImport values with the same name but different ordinals.
-                    if import.calling_convention != old_import.calling_convention {
-                        sess.dcx().emit_err(errors::MultipleExternalFuncDecl {
-                            span: import.span,
-                            function: import.name,
-                            library_name: &name,
-                        });
-                    }
-                }
-            }
-        }
-    }
-    sess.dcx().abort_if_errors();
-    dylib_table
-        .into_iter()
-        .map(|(name, imports)| {
-            (name, imports.into_iter().map(|(_, import)| import.clone()).collect())
-        })
-        .collect()
-}
-
-fn create_dll_import_libs<'a>(
-    sess: &Session,
-    archive_builder_builder: &dyn ArchiveBuilderBuilder,
-    used_libraries: impl IntoIterator<Item = &'a NativeLib>,
-    tmpdir: &Path,
-    is_direct_dependency: bool,
-) -> Vec<PathBuf> {
-    collate_raw_dylibs(sess, used_libraries)
-        .into_iter()
-        .map(|(raw_dylib_name, raw_dylib_imports)| {
-            let name_suffix = if is_direct_dependency { "_imports" } else { "_imports_indirect" };
-            let output_path = tmpdir.join(format!("{raw_dylib_name}{name_suffix}.lib"));
-
-            let mingw_gnu_toolchain = common::is_mingw_gnu_toolchain(&sess.target);
-
-            let items: Vec<ImportLibraryItem> = raw_dylib_imports
-                .iter()
-                .map(|import: &DllImport| {
-                    if sess.target.arch == "x86" {
-                        ImportLibraryItem {
-                            name: common::i686_decorated_name(
-                                import,
-                                mingw_gnu_toolchain,
-                                false,
-                                false,
-                            ),
-                            ordinal: import.ordinal(),
-                            symbol_name: import.is_missing_decorations().then(|| {
-                                common::i686_decorated_name(
-                                    import,
-                                    mingw_gnu_toolchain,
-                                    false,
-                                    true,
-                                )
-                            }),
-                            is_data: !import.is_fn,
-                        }
-                    } else {
-                        ImportLibraryItem {
-                            name: import.name.to_string(),
-                            ordinal: import.ordinal(),
-                            symbol_name: None,
-                            is_data: !import.is_fn,
-                        }
-                    }
-                })
-                .collect();
-
-            archive_builder_builder.create_dll_import_lib(
-                sess,
-                &raw_dylib_name,
-                items,
-                &output_path,
-            );
-
-            output_path
-        })
-        .collect()
-}
-
 /// Create a static archive.
 ///
 /// This is essentially the same thing as an rlib, but it also involves adding all of the upstream
@@ -1273,7 +1177,7 @@ mod win {
             let mut cp: u32 = 0;
             // We're using the `LOCALE_RETURN_NUMBER` flag to return a u32.
             // But the API requires us to pass the data as though it's a [u16] string.
-            let len = std::mem::size_of::<u32>() / std::mem::size_of::<u16>();
+            let len = size_of::<u32>() / size_of::<u16>();
             let data = std::slice::from_raw_parts_mut(&mut cp as *mut u32 as *mut u16, len);
             let len_written = GetLocaleInfoEx(
                 LOCALE_NAME_SYSTEM_DEFAULT,
@@ -2422,15 +2326,39 @@ fn linker_with_args(
         link_output_kind,
     );
 
+    // Raw-dylibs from all crates.
+    let raw_dylib_dir = tmpdir.join("raw-dylibs");
+    if sess.target.binary_format == BinaryFormat::Elf {
+        // On ELF we can't pass the raw-dylibs stubs to the linker as a path,
+        // instead we need to pass them via -l. To find the stub, we need to add
+        // the directory of the stub to the linker search path.
+        // We make an extra directory for this to avoid polluting the search path.
+        if let Err(error) = fs::create_dir(&raw_dylib_dir) {
+            sess.dcx().emit_fatal(errors::CreateTempDir { error })
+        }
+        cmd.include_path(&raw_dylib_dir);
+    }
+
     // Link with the import library generated for any raw-dylib functions.
-    for output_path in create_dll_import_libs(
-        sess,
-        archive_builder_builder,
-        codegen_results.crate_info.used_libraries.iter(),
-        tmpdir,
-        true,
-    ) {
-        cmd.add_object(&output_path);
+    if sess.target.is_like_windows {
+        for output_path in raw_dylib::create_raw_dylib_dll_import_libs(
+            sess,
+            archive_builder_builder,
+            codegen_results.crate_info.used_libraries.iter(),
+            tmpdir,
+            true,
+        ) {
+            cmd.add_object(&output_path);
+        }
+    } else {
+        for link_path in raw_dylib::create_raw_dylib_elf_stub_shared_objects(
+            sess,
+            codegen_results.crate_info.used_libraries.iter(),
+            &raw_dylib_dir,
+        ) {
+            // Always use verbatim linkage, see comments in create_raw_dylib_elf_stub_shared_objects.
+            cmd.link_dylib_by_name(&link_path, true, false);
+        }
     }
     // As with add_upstream_native_libraries, we need to add the upstream raw-dylib symbols in case
     // they are used within inlined functions or instantiated generic functions. We do this *after*
@@ -2449,19 +2377,35 @@ fn linker_with_args(
         .native_libraries
         .iter()
         .filter_map(|(&cnum, libraries)| {
-            (dependency_linkage[cnum] != Linkage::Static).then_some(libraries)
+            if sess.target.is_like_windows {
+                (dependency_linkage[cnum] != Linkage::Static).then_some(libraries)
+            } else {
+                Some(libraries)
+            }
         })
         .flatten()
         .collect::<Vec<_>>();
     native_libraries_from_nonstatics.sort_unstable_by(|a, b| a.name.as_str().cmp(b.name.as_str()));
-    for output_path in create_dll_import_libs(
-        sess,
-        archive_builder_builder,
-        native_libraries_from_nonstatics,
-        tmpdir,
-        false,
-    ) {
-        cmd.add_object(&output_path);
+
+    if sess.target.is_like_windows {
+        for output_path in raw_dylib::create_raw_dylib_dll_import_libs(
+            sess,
+            archive_builder_builder,
+            native_libraries_from_nonstatics,
+            tmpdir,
+            false,
+        ) {
+            cmd.add_object(&output_path);
+        }
+    } else {
+        for link_path in raw_dylib::create_raw_dylib_elf_stub_shared_objects(
+            sess,
+            native_libraries_from_nonstatics,
+            &raw_dylib_dir,
+        ) {
+            // Always use verbatim linkage, see comments in create_raw_dylib_elf_stub_shared_objects.
+            cmd.link_dylib_by_name(&link_path, true, false);
+        }
     }
 
     // Library linking above uses some global state for things like `-Bstatic`/`-Bdynamic` to make
@@ -3438,6 +3382,35 @@ fn add_lld_args(
     // this, `wasm-component-ld`, which is overridden if this option is passed.
     if !sess.target.is_like_wasm {
         cmd.cc_arg("-fuse-ld=lld");
+
+        // On ELF platforms like at least x64 linux, GNU ld and LLD have opposite defaults on some
+        // section garbage-collection features. For example, the somewhat popular `linkme` crate and
+        // its dependents rely in practice on this difference: when using lld, they need `-z
+        // nostart-stop-gc` to prevent encapsulation symbols and sections from being
+        // garbage-collected.
+        //
+        // More information about all this can be found in:
+        // - https://maskray.me/blog/2021-01-31-metadata-sections-comdat-and-shf-link-order
+        // - https://lld.llvm.org/ELF/start-stop-gc
+        //
+        // So when using lld, we restore, for now, the traditional behavior to help migration, but
+        // will remove it in the future.
+        // Since this only disables an optimization, it shouldn't create issues, but is in theory
+        // slightly suboptimal. However, it:
+        // - doesn't have any visible impact on our benchmarks
+        // - reduces the need to disable lld for the crates that depend on this
+        //
+        // Note that lld can detect some cases where this difference is relied on, and emits a
+        // dedicated error to add this link arg. We could make use of this error to emit an FCW. As
+        // of writing this, we don't do it, because lld is already enabled by default on nightly
+        // without this mitigation: no working project would see the FCW, so we do this to help
+        // stabilization.
+        //
+        // FIXME: emit an FCW if linking fails due its absence, and then remove this link-arg in the
+        // future.
+        if sess.target.llvm_target == "x86_64-unknown-linux-gnu" {
+            cmd.link_arg("-znostart-stop-gc");
+        }
     }
 
     if !flavor.is_gnu() {
diff --git a/compiler/rustc_codegen_ssa/src/back/link/raw_dylib.rs b/compiler/rustc_codegen_ssa/src/back/link/raw_dylib.rs
new file mode 100644
index 00000000000..2c24378afe1
--- /dev/null
+++ b/compiler/rustc_codegen_ssa/src/back/link/raw_dylib.rs
@@ -0,0 +1,373 @@
+use std::fs;
+use std::io::{BufWriter, Write};
+use std::path::{Path, PathBuf};
+
+use rustc_abi::Endian;
+use rustc_data_structures::base_n::{CASE_INSENSITIVE, ToBaseN};
+use rustc_data_structures::fx::FxIndexMap;
+use rustc_data_structures::stable_hasher::StableHasher;
+use rustc_hashes::Hash128;
+use rustc_session::Session;
+use rustc_session::cstore::DllImport;
+use rustc_session::utils::NativeLibKind;
+use rustc_span::Symbol;
+
+use crate::back::archive::ImportLibraryItem;
+use crate::back::link::ArchiveBuilderBuilder;
+use crate::errors::ErrorCreatingImportLibrary;
+use crate::{NativeLib, common, errors};
+
+/// Extract all symbols defined in raw-dylib libraries, collated by library name.
+///
+/// If we have multiple extern blocks that specify symbols defined in the same raw-dylib library,
+/// then the CodegenResults value contains one NativeLib instance for each block. However, the
+/// linker appears to expect only a single import library for each library used, so we need to
+/// collate the symbols together by library name before generating the import libraries.
+fn collate_raw_dylibs_windows<'a>(
+    sess: &Session,
+    used_libraries: impl IntoIterator<Item = &'a NativeLib>,
+) -> Vec<(String, Vec<DllImport>)> {
+    // Use index maps to preserve original order of imports and libraries.
+    let mut dylib_table = FxIndexMap::<String, FxIndexMap<Symbol, &DllImport>>::default();
+
+    for lib in used_libraries {
+        if lib.kind == NativeLibKind::RawDylib {
+            let ext = if lib.verbatim { "" } else { ".dll" };
+            let name = format!("{}{}", lib.name, ext);
+            let imports = dylib_table.entry(name.clone()).or_default();
+            for import in &lib.dll_imports {
+                if let Some(old_import) = imports.insert(import.name, import) {
+                    // FIXME: when we add support for ordinals, figure out if we need to do anything
+                    // if we have two DllImport values with the same name but different ordinals.
+                    if import.calling_convention != old_import.calling_convention {
+                        sess.dcx().emit_err(errors::MultipleExternalFuncDecl {
+                            span: import.span,
+                            function: import.name,
+                            library_name: &name,
+                        });
+                    }
+                }
+            }
+        }
+    }
+    sess.dcx().abort_if_errors();
+    dylib_table
+        .into_iter()
+        .map(|(name, imports)| {
+            (name, imports.into_iter().map(|(_, import)| import.clone()).collect())
+        })
+        .collect()
+}
+
+pub(super) fn create_raw_dylib_dll_import_libs<'a>(
+    sess: &Session,
+    archive_builder_builder: &dyn ArchiveBuilderBuilder,
+    used_libraries: impl IntoIterator<Item = &'a NativeLib>,
+    tmpdir: &Path,
+    is_direct_dependency: bool,
+) -> Vec<PathBuf> {
+    collate_raw_dylibs_windows(sess, used_libraries)
+        .into_iter()
+        .map(|(raw_dylib_name, raw_dylib_imports)| {
+            let name_suffix = if is_direct_dependency { "_imports" } else { "_imports_indirect" };
+            let output_path = tmpdir.join(format!("{raw_dylib_name}{name_suffix}.lib"));
+
+            let mingw_gnu_toolchain = common::is_mingw_gnu_toolchain(&sess.target);
+
+            let items: Vec<ImportLibraryItem> = raw_dylib_imports
+                .iter()
+                .map(|import: &DllImport| {
+                    if sess.target.arch == "x86" {
+                        ImportLibraryItem {
+                            name: common::i686_decorated_name(
+                                import,
+                                mingw_gnu_toolchain,
+                                false,
+                                false,
+                            ),
+                            ordinal: import.ordinal(),
+                            symbol_name: import.is_missing_decorations().then(|| {
+                                common::i686_decorated_name(
+                                    import,
+                                    mingw_gnu_toolchain,
+                                    false,
+                                    true,
+                                )
+                            }),
+                            is_data: !import.is_fn,
+                        }
+                    } else {
+                        ImportLibraryItem {
+                            name: import.name.to_string(),
+                            ordinal: import.ordinal(),
+                            symbol_name: None,
+                            is_data: !import.is_fn,
+                        }
+                    }
+                })
+                .collect();
+
+            archive_builder_builder.create_dll_import_lib(
+                sess,
+                &raw_dylib_name,
+                items,
+                &output_path,
+            );
+
+            output_path
+        })
+        .collect()
+}
+
+/// Extract all symbols defined in raw-dylib libraries, collated by library name.
+///
+/// If we have multiple extern blocks that specify symbols defined in the same raw-dylib library,
+/// then the CodegenResults value contains one NativeLib instance for each block. However, the
+/// linker appears to expect only a single import library for each library used, so we need to
+/// collate the symbols together by library name before generating the import libraries.
+fn collate_raw_dylibs_elf<'a>(
+    sess: &Session,
+    used_libraries: impl IntoIterator<Item = &'a NativeLib>,
+) -> Vec<(String, Vec<DllImport>)> {
+    // Use index maps to preserve original order of imports and libraries.
+    let mut dylib_table = FxIndexMap::<String, FxIndexMap<Symbol, &DllImport>>::default();
+
+    for lib in used_libraries {
+        if lib.kind == NativeLibKind::RawDylib {
+            let filename = if lib.verbatim {
+                lib.name.as_str().to_owned()
+            } else {
+                let ext = sess.target.dll_suffix.as_ref();
+                let prefix = sess.target.dll_prefix.as_ref();
+                format!("{prefix}{}{ext}", lib.name)
+            };
+
+            let imports = dylib_table.entry(filename.clone()).or_default();
+            for import in &lib.dll_imports {
+                imports.insert(import.name, import);
+            }
+        }
+    }
+    sess.dcx().abort_if_errors();
+    dylib_table
+        .into_iter()
+        .map(|(name, imports)| {
+            (name, imports.into_iter().map(|(_, import)| import.clone()).collect())
+        })
+        .collect()
+}
+
+pub(super) fn create_raw_dylib_elf_stub_shared_objects<'a>(
+    sess: &Session,
+    used_libraries: impl IntoIterator<Item = &'a NativeLib>,
+    raw_dylib_so_dir: &Path,
+) -> Vec<String> {
+    collate_raw_dylibs_elf(sess, used_libraries)
+        .into_iter()
+        .map(|(load_filename, raw_dylib_imports)| {
+            use std::hash::Hash;
+
+            // `load_filename` is the *target/loader* filename that will end up in NEEDED.
+            // Usually this will be something like `libc.so` or `libc.so.6` but with
+            // verbatim it might also be an absolute path.
+            // To be able to support this properly, we always put this load filename
+            // into the SONAME of the library and link it via a temporary file with a random name.
+            // This also avoids naming conflicts with non-raw-dylib linkage of the same library.
+
+            let shared_object = create_elf_raw_dylib_stub(sess, &load_filename, &raw_dylib_imports);
+
+            let mut file_name_hasher = StableHasher::new();
+            load_filename.hash(&mut file_name_hasher);
+            for raw_dylib in raw_dylib_imports {
+                raw_dylib.name.as_str().hash(&mut file_name_hasher);
+            }
+
+            let library_filename: Hash128 = file_name_hasher.finish();
+            let temporary_lib_name = format!(
+                "{}{}{}",
+                sess.target.dll_prefix,
+                library_filename.as_u128().to_base_fixed_len(CASE_INSENSITIVE),
+                sess.target.dll_suffix
+            );
+            let link_path = raw_dylib_so_dir.join(&temporary_lib_name);
+
+            let file = match fs::File::create_new(&link_path) {
+                Ok(file) => file,
+                Err(error) => sess.dcx().emit_fatal(ErrorCreatingImportLibrary {
+                    lib_name: &load_filename,
+                    error: error.to_string(),
+                }),
+            };
+            if let Err(error) = BufWriter::new(file).write_all(&shared_object) {
+                sess.dcx().emit_fatal(ErrorCreatingImportLibrary {
+                    lib_name: &load_filename,
+                    error: error.to_string(),
+                });
+            };
+
+            temporary_lib_name
+        })
+        .collect()
+}
+
+/// Create an ELF .so stub file for raw-dylib.
+/// It exports all the provided symbols, but is otherwise empty.
+fn create_elf_raw_dylib_stub(sess: &Session, soname: &str, symbols: &[DllImport]) -> Vec<u8> {
+    use object::write::elf as write;
+    use object::{Architecture, elf};
+
+    let mut stub_buf = Vec::new();
+
+    // Build the stub ELF using the object crate.
+    // The high-level portable API does not allow for the fine-grained control we need,
+    // so this uses the low-level object::write::elf API.
+    // The low-level API consists of two stages: reservation and writing.
+    // We first reserve space for all the things in the binary and then write them.
+    // It is important that the order of reservation matches the order of writing.
+    // The object crate contains many debug asserts that fire if you get this wrong.
+
+    let endianness = match sess.target.options.endian {
+        Endian::Little => object::Endianness::Little,
+        Endian::Big => object::Endianness::Big,
+    };
+    let mut stub = write::Writer::new(endianness, true, &mut stub_buf);
+
+    // These initial reservations don't reserve any bytes in the binary yet,
+    // they just allocate in the internal data structures.
+
+    // First, we crate the dynamic symbol table. It starts with a null symbol
+    // and then all the symbols and their dynamic strings.
+    stub.reserve_null_dynamic_symbol_index();
+
+    let dynstrs = symbols
+        .iter()
+        .map(|sym| {
+            stub.reserve_dynamic_symbol_index();
+            (sym, stub.add_dynamic_string(sym.name.as_str().as_bytes()))
+        })
+        .collect::<Vec<_>>();
+
+    let soname = stub.add_dynamic_string(soname.as_bytes());
+
+    // Reserve the sections.
+    // We have the minimal sections for a dynamic SO and .text where we point our dummy symbols to.
+    stub.reserve_shstrtab_section_index();
+    let text_section_name = stub.add_section_name(".text".as_bytes());
+    let text_section = stub.reserve_section_index();
+    stub.reserve_dynstr_section_index();
+    stub.reserve_dynsym_section_index();
+    stub.reserve_dynamic_section_index();
+
+    // These reservations now determine the actual layout order of the object file.
+    stub.reserve_file_header();
+    stub.reserve_shstrtab();
+    stub.reserve_section_headers();
+    stub.reserve_dynstr();
+    stub.reserve_dynsym();
+    stub.reserve_dynamic(2); // DT_SONAME, DT_NULL
+
+    // First write the ELF header with the arch information.
+    let Some((arch, sub_arch)) = sess.target.object_architecture(&sess.unstable_target_features)
+    else {
+        sess.dcx().fatal(format!(
+            "raw-dylib is not supported for the architecture `{}`",
+            sess.target.arch
+        ));
+    };
+    let e_machine = match (arch, sub_arch) {
+        (Architecture::Aarch64, None) => elf::EM_AARCH64,
+        (Architecture::Aarch64_Ilp32, None) => elf::EM_AARCH64,
+        (Architecture::Arm, None) => elf::EM_ARM,
+        (Architecture::Avr, None) => elf::EM_AVR,
+        (Architecture::Bpf, None) => elf::EM_BPF,
+        (Architecture::Csky, None) => elf::EM_CSKY,
+        (Architecture::E2K32, None) => elf::EM_MCST_ELBRUS,
+        (Architecture::E2K64, None) => elf::EM_MCST_ELBRUS,
+        (Architecture::I386, None) => elf::EM_386,
+        (Architecture::X86_64, None) => elf::EM_X86_64,
+        (Architecture::X86_64_X32, None) => elf::EM_X86_64,
+        (Architecture::Hexagon, None) => elf::EM_HEXAGON,
+        (Architecture::LoongArch64, None) => elf::EM_LOONGARCH,
+        (Architecture::M68k, None) => elf::EM_68K,
+        (Architecture::Mips, None) => elf::EM_MIPS,
+        (Architecture::Mips64, None) => elf::EM_MIPS,
+        (Architecture::Mips64_N32, None) => elf::EM_MIPS,
+        (Architecture::Msp430, None) => elf::EM_MSP430,
+        (Architecture::PowerPc, None) => elf::EM_PPC,
+        (Architecture::PowerPc64, None) => elf::EM_PPC64,
+        (Architecture::Riscv32, None) => elf::EM_RISCV,
+        (Architecture::Riscv64, None) => elf::EM_RISCV,
+        (Architecture::S390x, None) => elf::EM_S390,
+        (Architecture::Sbf, None) => elf::EM_SBF,
+        (Architecture::Sharc, None) => elf::EM_SHARC,
+        (Architecture::Sparc, None) => elf::EM_SPARC,
+        (Architecture::Sparc32Plus, None) => elf::EM_SPARC32PLUS,
+        (Architecture::Sparc64, None) => elf::EM_SPARCV9,
+        (Architecture::Xtensa, None) => elf::EM_XTENSA,
+        _ => {
+            sess.dcx().fatal(format!(
+                "raw-dylib is not supported for the architecture `{}`",
+                sess.target.arch
+            ));
+        }
+    };
+
+    stub.write_file_header(&write::FileHeader {
+        os_abi: crate::back::metadata::elf_os_abi(sess),
+        abi_version: 0,
+        e_type: object::elf::ET_DYN,
+        e_machine,
+        e_entry: 0,
+        e_flags: crate::back::metadata::elf_e_flags(arch, sess),
+    })
+    .unwrap();
+
+    // .shstrtab
+    stub.write_shstrtab();
+
+    // Section headers
+    stub.write_null_section_header();
+    stub.write_shstrtab_section_header();
+    // Create a dummy .text section for our dummy symbols.
+    stub.write_section_header(&write::SectionHeader {
+        name: Some(text_section_name),
+        sh_type: elf::SHT_PROGBITS,
+        sh_flags: 0,
+        sh_addr: 0,
+        sh_offset: 0,
+        sh_size: 0,
+        sh_link: 0,
+        sh_info: 0,
+        sh_addralign: 1,
+        sh_entsize: 0,
+    });
+    stub.write_dynstr_section_header(0);
+    stub.write_dynsym_section_header(0, 1);
+    stub.write_dynamic_section_header(0);
+
+    // .dynstr
+    stub.write_dynstr();
+
+    // .dynsym
+    stub.write_null_dynamic_symbol();
+    for (_, name) in dynstrs {
+        stub.write_dynamic_symbol(&write::Sym {
+            name: Some(name),
+            st_info: (elf::STB_GLOBAL << 4) | elf::STT_NOTYPE,
+            st_other: elf::STV_DEFAULT,
+            section: Some(text_section),
+            st_shndx: 0, // ignored by object in favor of the `section` field
+            st_value: 0,
+            st_size: 0,
+        });
+    }
+
+    // .dynamic
+    // the DT_SONAME will be used by the linker to populate DT_NEEDED
+    // which the loader uses to find the library.
+    // DT_NULL terminates the .dynamic table.
+    stub.write_dynamic_string(elf::DT_SONAME, soname);
+    stub.write_dynamic(elf::DT_NULL, 0);
+
+    stub_buf
+}
diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs
index 818edaf6603..a8405a2aec9 100644
--- a/compiler/rustc_codegen_ssa/src/back/linker.rs
+++ b/compiler/rustc_codegen_ssa/src/back/linker.rs
@@ -1655,9 +1655,9 @@ impl<'a> Linker for AixLinker<'a> {
         }
     }
 
-    fn link_dylib_by_name(&mut self, name: &str, _verbatim: bool, _as_needed: bool) {
+    fn link_dylib_by_name(&mut self, name: &str, verbatim: bool, _as_needed: bool) {
         self.hint_dynamic();
-        self.link_or_cc_arg(format!("-l{name}"));
+        self.link_or_cc_arg(if verbatim { String::from(name) } else { format!("-l{name}") });
     }
 
     fn link_dylib_by_path(&mut self, path: &Path, _as_needed: bool) {
@@ -1668,7 +1668,7 @@ impl<'a> Linker for AixLinker<'a> {
     fn link_staticlib_by_name(&mut self, name: &str, verbatim: bool, whole_archive: bool) {
         self.hint_static();
         if !whole_archive {
-            self.link_or_cc_arg(format!("-l{name}"));
+            self.link_or_cc_arg(if verbatim { String::from(name) } else { format!("-l{name}") });
         } else {
             let mut arg = OsString::from("-bkeepfile:");
             arg.push(find_native_static_library(name, verbatim, self.sess));
diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs
index 3aacbcde1a7..68b453ff424 100644
--- a/compiler/rustc_codegen_ssa/src/back/metadata.rs
+++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs
@@ -9,8 +9,7 @@ use itertools::Itertools;
 use object::write::{self, StandardSegment, Symbol, SymbolSection};
 use object::{
     Architecture, BinaryFormat, Endianness, FileFlags, Object, ObjectSection, ObjectSymbol,
-    SectionFlags, SectionKind, SubArchitecture, SymbolFlags, SymbolKind, SymbolScope, elf, pe,
-    xcoff,
+    SectionFlags, SectionKind, SymbolFlags, SymbolKind, SymbolScope, elf, pe, xcoff,
 };
 use rustc_abi::Endian;
 use rustc_data_structures::memmap::Mmap;
@@ -206,51 +205,10 @@ pub(crate) fn create_object_file(sess: &Session) -> Option<write::Object<'static
         Endian::Little => Endianness::Little,
         Endian::Big => Endianness::Big,
     };
-    let (architecture, sub_architecture) = match &sess.target.arch[..] {
-        "arm" => (Architecture::Arm, None),
-        "aarch64" => (
-            if sess.target.pointer_width == 32 {
-                Architecture::Aarch64_Ilp32
-            } else {
-                Architecture::Aarch64
-            },
-            None,
-        ),
-        "x86" => (Architecture::I386, None),
-        "s390x" => (Architecture::S390x, None),
-        "mips" | "mips32r6" => (Architecture::Mips, None),
-        "mips64" | "mips64r6" => (Architecture::Mips64, None),
-        "x86_64" => (
-            if sess.target.pointer_width == 32 {
-                Architecture::X86_64_X32
-            } else {
-                Architecture::X86_64
-            },
-            None,
-        ),
-        "powerpc" => (Architecture::PowerPc, None),
-        "powerpc64" => (Architecture::PowerPc64, None),
-        "riscv32" => (Architecture::Riscv32, None),
-        "riscv64" => (Architecture::Riscv64, None),
-        "sparc" => {
-            if sess.unstable_target_features.contains(&sym::v8plus) {
-                // Target uses V8+, aka EM_SPARC32PLUS, aka 64-bit V9 but in 32-bit mode
-                (Architecture::Sparc32Plus, None)
-            } else {
-                // Target uses V7 or V8, aka EM_SPARC
-                (Architecture::Sparc, None)
-            }
-        }
-        "sparc64" => (Architecture::Sparc64, None),
-        "avr" => (Architecture::Avr, None),
-        "msp430" => (Architecture::Msp430, None),
-        "hexagon" => (Architecture::Hexagon, None),
-        "bpf" => (Architecture::Bpf, None),
-        "loongarch64" => (Architecture::LoongArch64, None),
-        "csky" => (Architecture::Csky, None),
-        "arm64ec" => (Architecture::Aarch64, Some(SubArchitecture::Arm64EC)),
-        // Unsupported architecture.
-        _ => return None,
+    let Some((architecture, sub_architecture)) =
+        sess.target.object_architecture(&sess.unstable_target_features)
+    else {
+        return None;
     };
     let binary_format = sess.target.binary_format.to_object();
 
@@ -292,7 +250,26 @@ pub(crate) fn create_object_file(sess: &Session) -> Option<write::Object<'static
 
         file.set_mangling(original_mangling);
     }
-    let e_flags = match architecture {
+    let e_flags = elf_e_flags(architecture, sess);
+    // adapted from LLVM's `MCELFObjectTargetWriter::getOSABI`
+    let os_abi = elf_os_abi(sess);
+    let abi_version = 0;
+    add_gnu_property_note(&mut file, architecture, binary_format, endianness);
+    file.flags = FileFlags::Elf { os_abi, abi_version, e_flags };
+    Some(file)
+}
+
+pub(super) fn elf_os_abi(sess: &Session) -> u8 {
+    match sess.target.options.os.as_ref() {
+        "hermit" => elf::ELFOSABI_STANDALONE,
+        "freebsd" => elf::ELFOSABI_FREEBSD,
+        "solaris" => elf::ELFOSABI_SOLARIS,
+        _ => elf::ELFOSABI_NONE,
+    }
+}
+
+pub(super) fn elf_e_flags(architecture: Architecture, sess: &Session) -> u32 {
+    match architecture {
         Architecture::Mips => {
             let arch = match sess.target.options.cpu.as_ref() {
                 "mips1" => elf::EF_MIPS_ARCH_1,
@@ -387,18 +364,7 @@ pub(crate) fn create_object_file(sess: &Session) -> Option<write::Object<'static
             e_flags
         }
         _ => 0,
-    };
-    // adapted from LLVM's `MCELFObjectTargetWriter::getOSABI`
-    let os_abi = match sess.target.options.os.as_ref() {
-        "hermit" => elf::ELFOSABI_STANDALONE,
-        "freebsd" => elf::ELFOSABI_FREEBSD,
-        "solaris" => elf::ELFOSABI_SOLARIS,
-        _ => elf::ELFOSABI_NONE,
-    };
-    let abi_version = 0;
-    add_gnu_property_note(&mut file, architecture, binary_format, endianness);
-    file.flags = FileFlags::Elf { os_abi, abi_version, e_flags };
-    Some(file)
+    }
 }
 
 /// Mach-O files contain information about:
diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs
index c8bb229998e..87992ce2e11 100644
--- a/compiler/rustc_codegen_ssa/src/back/write.rs
+++ b/compiler/rustc_codegen_ssa/src/back/write.rs
@@ -6,6 +6,7 @@ use std::sync::Arc;
 use std::sync::mpsc::{Receiver, Sender, channel};
 use std::{fs, io, mem, str, thread};
 
+use rustc_abi::Size;
 use rustc_ast::attr;
 use rustc_ast::expand::autodiff_attrs::AutoDiffItem;
 use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
@@ -355,6 +356,7 @@ pub struct CodegenContext<B: WriteBackendMethods> {
     pub target_is_like_aix: bool,
     pub split_debuginfo: rustc_target::spec::SplitDebuginfo,
     pub split_dwarf_kind: rustc_session::config::SplitDwarfKind,
+    pub pointer_size: Size,
 
     /// All commandline args used to invoke the compiler, with @file args fully expanded.
     /// This will only be used within debug info, e.g. in the pdb file on windows
@@ -1216,6 +1218,7 @@ fn start_executing_work<B: ExtraBackendMethods>(
         split_debuginfo: tcx.sess.split_debuginfo(),
         split_dwarf_kind: tcx.sess.opts.unstable_opts.split_dwarf_kind,
         parallel: backend.supports_parallel() && !sess.opts.unstable_opts.no_parallel_backend,
+        pointer_size: tcx.data_layout.pointer_size,
     };
 
     // This is the "main loop" of parallel work happening for parallel codegen.
diff --git a/compiler/rustc_codegen_ssa/src/errors.rs b/compiler/rustc_codegen_ssa/src/errors.rs
index 7e28961599f..ccf6d12977f 100644
--- a/compiler/rustc_codegen_ssa/src/errors.rs
+++ b/compiler/rustc_codegen_ssa/src/errors.rs
@@ -957,6 +957,7 @@ pub enum InvalidMonomorphization<'tcx> {
     },
 
     #[diag(codegen_ssa_invalid_monomorphization_mask_type, code = E0511)]
+    #[note]
     MaskType {
         #[primary_span]
         span: Span,
diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs
index 4e758bfdec3..8ad04032414 100644
--- a/compiler/rustc_codegen_ssa/src/lib.rs
+++ b/compiler/rustc_codegen_ssa/src/lib.rs
@@ -14,7 +14,6 @@
 #![feature(rustdoc_internals)]
 #![feature(trait_alias)]
 #![feature(try_blocks)]
-#![warn(unreachable_pub)]
 // tidy-alphabetical-end
 
 //! This crate contains codegen code that is used by all codegen backends (LLVM and others).
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index e2a9b540d30..6d1930a402d 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -3,6 +3,7 @@ use std::cmp;
 use rustc_abi::{BackendRepr, ExternAbi, HasDataLayout, Reg, WrappingRange};
 use rustc_ast as ast;
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
+use rustc_data_structures::packed::Pu128;
 use rustc_hir::lang_items::LangItem;
 use rustc_middle::mir::{self, AssertKind, InlineAsmMacro, SwitchTargets, UnwindTerminateReason};
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, ValidityRequirement};
@@ -406,6 +407,39 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 let cmp = bx.icmp(IntPredicate::IntEQ, discr_value, llval);
                 bx.cond_br_with_expect(cmp, lltarget, llotherwise, expect);
             }
+        } else if target_iter.len() == 2
+            && self.mir[targets.otherwise()].is_empty_unreachable()
+            && targets.all_values().contains(&Pu128(0))
+            && targets.all_values().contains(&Pu128(1))
+        {
+            // This is the really common case for `bool`, `Option`, etc.
+            // By using `trunc nuw` we communicate that other values are
+            // impossible without needing `switch` or `assume`s.
+            let true_bb = targets.target_for_value(1);
+            let false_bb = targets.target_for_value(0);
+            let true_ll = helper.llbb_with_cleanup(self, true_bb);
+            let false_ll = helper.llbb_with_cleanup(self, false_bb);
+
+            let expected_cond_value = if self.cx.sess().opts.optimize == OptLevel::No {
+                None
+            } else {
+                match (self.cold_blocks[true_bb], self.cold_blocks[false_bb]) {
+                    // Same coldness, no expectation
+                    (true, true) | (false, false) => None,
+                    // Different coldness, expect the non-cold one
+                    (true, false) => Some(false),
+                    (false, true) => Some(true),
+                }
+            };
+
+            let bool_ty = bx.tcx().types.bool;
+            let cond = if switch_ty == bool_ty {
+                discr_value
+            } else {
+                let bool_llty = bx.immediate_backend_type(bx.layout_of(bool_ty));
+                bx.unchecked_utrunc(discr_value, bool_llty)
+            };
+            bx.cond_br_with_expect(cond, true_ll, false_ll, expected_cond_value);
         } else if self.cx.sess().opts.optimize == OptLevel::No
             && target_iter.len() == 2
             && self.mir[targets.otherwise()].is_empty_unreachable()
diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs
index 4b33bdeadba..acae09b2c25 100644
--- a/compiler/rustc_codegen_ssa/src/mir/operand.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs
@@ -203,30 +203,14 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
         let alloc_align = alloc.inner().align;
         assert!(alloc_align >= layout.align.abi);
 
-        // Returns `None` when the value is partially undefined or any byte of it has provenance.
-        // Otherwise returns the value or (if the entire value is undef) returns an undef.
         let read_scalar = |start, size, s: abi::Scalar, ty| {
-            let range = alloc_range(start, size);
             match alloc.0.read_scalar(
                 bx,
-                range,
+                alloc_range(start, size),
                 /*read_provenance*/ matches!(s.primitive(), abi::Primitive::Pointer(_)),
             ) {
-                Ok(val) => Some(bx.scalar_to_backend(val, s, ty)),
-                Err(_) => {
-                    // We may have failed due to partial provenance or unexpected provenance,
-                    // continue down the normal code path if so.
-                    if alloc.0.provenance().range_empty(range, &bx.tcx())
-                        // Since `read_scalar` failed, but there were no relocations involved, the
-                        // bytes must be partially or fully uninitialized. Thus we can now unwrap the
-                        // information about the range of uninit bytes and check if it's the full range.
-                        && alloc.0.init_mask().is_range_initialized(range).unwrap_err() == range
-                    {
-                        Some(bx.const_undef(ty))
-                    } else {
-                        None
-                    }
-                }
+                Ok(val) => bx.scalar_to_backend(val, s, ty),
+                Err(_) => bx.const_poison(ty),
             }
         };
 
@@ -237,14 +221,16 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
         // check that walks over the type of `mplace` to make sure it is truly correct to treat this
         // like a `Scalar` (or `ScalarPair`).
         match layout.backend_repr {
-            BackendRepr::Scalar(s) => {
+            BackendRepr::Scalar(s @ abi::Scalar::Initialized { .. }) => {
                 let size = s.size(bx);
                 assert_eq!(size, layout.size, "abi::Scalar size does not match layout size");
-                if let Some(val) = read_scalar(offset, size, s, bx.immediate_backend_type(layout)) {
-                    return OperandRef { val: OperandValue::Immediate(val), layout };
-                }
+                let val = read_scalar(offset, size, s, bx.immediate_backend_type(layout));
+                OperandRef { val: OperandValue::Immediate(val), layout }
             }
-            BackendRepr::ScalarPair(a, b) => {
+            BackendRepr::ScalarPair(
+                a @ abi::Scalar::Initialized { .. },
+                b @ abi::Scalar::Initialized { .. },
+            ) => {
                 let (a_size, b_size) = (a.size(bx), b.size(bx));
                 let b_offset = (offset + a_size).align_to(b.align(bx).abi);
                 assert!(b_offset.bytes() > 0);
@@ -260,21 +246,20 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
                     b,
                     bx.scalar_pair_element_backend_type(layout, 1, true),
                 );
-                if let (Some(a_val), Some(b_val)) = (a_val, b_val) {
-                    return OperandRef { val: OperandValue::Pair(a_val, b_val), layout };
-                }
+                OperandRef { val: OperandValue::Pair(a_val, b_val), layout }
+            }
+            _ if layout.is_zst() => OperandRef::zero_sized(layout),
+            _ => {
+                // Neither a scalar nor scalar pair. Load from a place
+                // FIXME: should we cache `const_data_from_alloc` to avoid repeating this for the
+                // same `ConstAllocation`?
+                let init = bx.const_data_from_alloc(alloc);
+                let base_addr = bx.static_addr_of(init, alloc_align, None);
+
+                let llval = bx.const_ptr_byte_offset(base_addr, offset);
+                bx.load_operand(PlaceRef::new_sized(llval, layout))
             }
-            _ if layout.is_zst() => return OperandRef::zero_sized(layout),
-            _ => {}
         }
-        // Neither a scalar nor scalar pair. Load from a place
-        // FIXME: should we cache `const_data_from_alloc` to avoid repeating this for the
-        // same `ConstAllocation`?
-        let init = bx.const_data_from_alloc(alloc);
-        let base_addr = bx.static_addr_of(init, alloc_align, None);
-
-        let llval = bx.const_ptr_byte_offset(base_addr, offset);
-        bx.load_operand(PlaceRef::new_sized(llval, layout))
     }
 
     /// Asserts that this operand refers to a scalar and returns
diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs
index 00e01e47fee..6988724b421 100644
--- a/compiler/rustc_codegen_ssa/src/mir/place.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/place.rs
@@ -133,7 +133,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
         Self::alloca(bx, ptr_layout)
     }
 
-    pub fn len<Cx: ConstCodegenMethods<'tcx, Value = V>>(&self, cx: &Cx) -> V {
+    pub fn len<Cx: ConstCodegenMethods<Value = V>>(&self, cx: &Cx) -> V {
         if let FieldsShape::Array { count, .. } = self.layout.fields {
             if self.layout.is_unsized() {
                 assert_eq!(count, 0);
diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
index 67555d40332..95b108b1d33 100644
--- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
@@ -8,7 +8,7 @@ use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
 use rustc_middle::{bug, mir, span_bug};
 use rustc_session::config::OptLevel;
 use rustc_span::{DUMMY_SP, Span};
-use tracing::{debug, instrument, trace};
+use tracing::{debug, instrument};
 
 use super::operand::{OperandRef, OperandValue};
 use super::place::PlaceRef;
@@ -93,8 +93,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     return;
                 }
 
-                // If `v` is an integer constant whose value is just a single byte repeated N times,
-                // emit a `memset` filling the entire `dest` with that byte.
                 let try_init_all_same = |bx: &mut Bx, v| {
                     let start = dest.val.llval;
                     let size = bx.const_usize(dest.layout.size.bytes());
@@ -119,33 +117,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     false
                 };
 
-                trace!(?cg_elem.val);
                 match cg_elem.val {
                     OperandValue::Immediate(v) => {
                         if try_init_all_same(bx, v) {
                             return;
                         }
                     }
-                    OperandValue::Pair(a, b) => {
-                        let a_is_undef = bx.cx().is_undef(a);
-                        match (a_is_undef, bx.cx().is_undef(b)) {
-                            // Can happen for uninit unions
-                            (true, true) => {
-                                // FIXME: can we produce better output here?
-                            }
-                            (false, true) | (true, false) => {
-                                let val = if a_is_undef { b } else { a };
-                                if try_init_all_same(bx, val) {
-                                    return;
-                                }
-                            }
-                            (false, false) => {
-                                // FIXME: if both are the same value, use try_init_all_same
-                            }
-                        }
-                    }
-                    OperandValue::ZeroSized => unreachable!("checked above"),
-                    OperandValue::Ref(..) => {}
+                    _ => (),
                 }
 
                 let count = self
@@ -386,6 +364,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     ) -> Bx::Value {
         assert_eq!(from_scalar.size(self.cx), to_scalar.size(self.cx));
 
+        // While optimizations will remove no-op transmutes, they might still be
+        // there in debug or things that aren't no-op in MIR because they change
+        // the Rust type but not the underlying layout/niche.
+        if from_scalar == to_scalar && from_backend_ty == to_backend_ty {
+            return imm;
+        }
+
         use abi::Primitive::*;
         imm = bx.from_immediate(imm);
 
diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs
index d8b9bdb55da..95a5e96fe46 100644
--- a/compiler/rustc_codegen_ssa/src/target_features.rs
+++ b/compiler/rustc_codegen_ssa/src/target_features.rs
@@ -10,7 +10,7 @@ use rustc_middle::query::Providers;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::parse::feature_err;
 use rustc_span::{Span, Symbol, sym};
-use rustc_target::target_features;
+use rustc_target::target_features::{self, Stability};
 
 use crate::errors;
 
@@ -87,12 +87,17 @@ pub(crate) fn from_target_feature_attr(
                     // But ensure the ABI does not forbid enabling this.
                     // Here we do assume that LLVM doesn't add even more implied features
                     // we don't know about, at least no features that would have ABI effects!
-                    if abi_feature_constraints.incompatible.contains(&name.as_str()) {
-                        tcx.dcx().emit_err(errors::ForbiddenTargetFeatureAttr {
-                            span: item.span(),
-                            feature: name.as_str(),
-                            reason: "this feature is incompatible with the target ABI",
-                        });
+                    // We skip this logic in rustdoc, where we want to allow all target features of
+                    // all targets, so we can't check their ABI compatibility and anyway we are not
+                    // generating code so "it's fine".
+                    if !tcx.sess.opts.actually_rustdoc {
+                        if abi_feature_constraints.incompatible.contains(&name.as_str()) {
+                            tcx.dcx().emit_err(errors::ForbiddenTargetFeatureAttr {
+                                span: item.span(),
+                                feature: name.as_str(),
+                                reason: "this feature is incompatible with the target ABI",
+                            });
+                        }
                     }
                     target_features.push(TargetFeature { name, implied: name != feature_sym })
                 }
@@ -142,11 +147,38 @@ pub(crate) fn provide(providers: &mut Providers) {
         rust_target_features: |tcx, cnum| {
             assert_eq!(cnum, LOCAL_CRATE);
             if tcx.sess.opts.actually_rustdoc {
-                // rustdoc needs to be able to document functions that use all the features, so
-                // whitelist them all
-                rustc_target::target_features::all_rust_features()
-                    .map(|(a, b)| (a.to_string(), b))
-                    .collect()
+                // HACK: rustdoc would like to pretend that we have all the target features, so we
+                // have to merge all the lists into one. To ensure an unstable target never prevents
+                // a stable one from working, we merge the stability info of all instances of the
+                // same target feature name, with the "most stable" taking precedence. And then we
+                // hope that this doesn't cause issues anywhere else in the compiler...
+                let mut result: UnordMap<String, Stability> = Default::default();
+                for (name, stability) in rustc_target::target_features::all_rust_features() {
+                    use std::collections::hash_map::Entry;
+                    match result.entry(name.to_owned()) {
+                        Entry::Vacant(vacant_entry) => {
+                            vacant_entry.insert(stability);
+                        }
+                        Entry::Occupied(mut occupied_entry) => {
+                            // Merge the two stabilities, "more stable" taking precedence.
+                            match (occupied_entry.get(), stability) {
+                                (Stability::Stable, _)
+                                | (
+                                    Stability::Unstable { .. },
+                                    Stability::Unstable { .. } | Stability::Forbidden { .. },
+                                )
+                                | (Stability::Forbidden { .. }, Stability::Forbidden { .. }) => {
+                                    // The stability in the entry is at least as good as the new one, just keep it.
+                                }
+                                _ => {
+                                    // Overwrite stabilite.
+                                    occupied_entry.insert(stability);
+                                }
+                            }
+                        }
+                    }
+                }
+                result
             } else {
                 tcx.sess
                     .target
diff --git a/compiler/rustc_codegen_ssa/src/traits/abi.rs b/compiler/rustc_codegen_ssa/src/traits/abi.rs
index 60d8f2a9ece..49c51caa996 100644
--- a/compiler/rustc_codegen_ssa/src/traits/abi.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/abi.rs
@@ -1,5 +1,5 @@
 use super::BackendTypes;
 
-pub trait AbiBuilderMethods<'tcx>: BackendTypes {
+pub trait AbiBuilderMethods: BackendTypes {
     fn get_param(&mut self, index: usize) -> Self::Value;
 }
diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs
index 99fd6b6510f..5f91133d5b4 100644
--- a/compiler/rustc_codegen_ssa/src/traits/builder.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs
@@ -40,7 +40,7 @@ pub trait BuilderMethods<'a, 'tcx>:
     + CoverageInfoBuilderMethods<'tcx>
     + DebugInfoBuilderMethods
     + ArgAbiBuilderMethods<'tcx>
-    + AbiBuilderMethods<'tcx>
+    + AbiBuilderMethods
     + IntrinsicCallBuilderMethods<'tcx>
     + AsmBuilderMethods<'tcx>
     + StaticBuilderMethods
diff --git a/compiler/rustc_codegen_ssa/src/traits/consts.rs b/compiler/rustc_codegen_ssa/src/traits/consts.rs
index 5cfb56ebace..d83a04d814b 100644
--- a/compiler/rustc_codegen_ssa/src/traits/consts.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/consts.rs
@@ -3,13 +3,12 @@ use rustc_middle::mir::interpret::{ConstAllocation, Scalar};
 
 use super::BackendTypes;
 
-pub trait ConstCodegenMethods<'tcx>: BackendTypes {
+pub trait ConstCodegenMethods: BackendTypes {
     // Constant constructors
     fn const_null(&self, t: Self::Type) -> Self::Value;
     /// Generate an uninitialized value (matching uninitialized memory in MIR).
     /// Whether memory is initialized or not is tracked byte-for-byte.
     fn const_undef(&self, t: Self::Type) -> Self::Value;
-    fn is_undef(&self, v: Self::Value) -> bool;
     /// Generate a fake value. Poison always affects the entire value, even if just a single byte is
     /// poison. This can only be used in codepaths that are already UB, i.e., UB-free Rust code
     /// (including code that e.g. copies uninit memory with `MaybeUninit`) can never encounter a
@@ -38,7 +37,7 @@ pub trait ConstCodegenMethods<'tcx>: BackendTypes {
     fn const_to_opt_uint(&self, v: Self::Value) -> Option<u64>;
     fn const_to_opt_u128(&self, v: Self::Value, sign_ext: bool) -> Option<u128>;
 
-    fn const_data_from_alloc(&self, alloc: ConstAllocation<'tcx>) -> Self::Value;
+    fn const_data_from_alloc(&self, alloc: ConstAllocation<'_>) -> Self::Value;
 
     fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: Self::Type) -> Self::Value;
 
diff --git a/compiler/rustc_codegen_ssa/src/traits/mod.rs b/compiler/rustc_codegen_ssa/src/traits/mod.rs
index 90fcfbe4da7..239857a4298 100644
--- a/compiler/rustc_codegen_ssa/src/traits/mod.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/mod.rs
@@ -55,7 +55,7 @@ pub trait CodegenObject = Copy + PartialEq + fmt::Debug;
 pub trait CodegenMethods<'tcx> = LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>>
     + FnAbiOf<'tcx, FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>>
     + TypeCodegenMethods<'tcx>
-    + ConstCodegenMethods<'tcx>
+    + ConstCodegenMethods
     + StaticCodegenMethods
     + DebugInfoCodegenMethods<'tcx>
     + AsmCodegenMethods<'tcx>
diff --git a/compiler/rustc_codegen_ssa/src/traits/type_.rs b/compiler/rustc_codegen_ssa/src/traits/type_.rs
index fbd927d0d66..32d9f27d32d 100644
--- a/compiler/rustc_codegen_ssa/src/traits/type_.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/type_.rs
@@ -9,7 +9,7 @@ use super::misc::MiscCodegenMethods;
 use crate::common::TypeKind;
 use crate::mir::place::PlaceRef;
 
-pub trait BaseTypeCodegenMethods<'tcx>: BackendTypes {
+pub trait BaseTypeCodegenMethods: BackendTypes {
     fn type_i8(&self) -> Self::Type;
     fn type_i16(&self) -> Self::Type;
     fn type_i32(&self) -> Self::Type;
@@ -41,7 +41,7 @@ pub trait BaseTypeCodegenMethods<'tcx>: BackendTypes {
 }
 
 pub trait DerivedTypeCodegenMethods<'tcx>:
-    BaseTypeCodegenMethods<'tcx> + MiscCodegenMethods<'tcx> + HasTyCtxt<'tcx> + HasTypingEnv<'tcx>
+    BaseTypeCodegenMethods + MiscCodegenMethods<'tcx> + HasTyCtxt<'tcx> + HasTypingEnv<'tcx>
 {
     fn type_int(&self) -> Self::Type {
         match &self.sess().target.c_int_width[..] {
@@ -87,10 +87,7 @@ pub trait DerivedTypeCodegenMethods<'tcx>:
 }
 
 impl<'tcx, T> DerivedTypeCodegenMethods<'tcx> for T where
-    Self: BaseTypeCodegenMethods<'tcx>
-        + MiscCodegenMethods<'tcx>
-        + HasTyCtxt<'tcx>
-        + HasTypingEnv<'tcx>
+    Self: BaseTypeCodegenMethods + MiscCodegenMethods<'tcx> + HasTyCtxt<'tcx> + HasTypingEnv<'tcx>
 {
 }