Merge remote-tracking branch 'origin/master' into mpk/add-long-error-message-for-E0311

19 files changed, 433 insertions, 320 deletions

diff --git a/compiler/rustc_codegen_ssa/src/back/archive.rs b/compiler/rustc_codegen_ssa/src/back/archive.rs
index 0d2aa483d3d..bad58d0a8a0 100644
--- a/compiler/rustc_codegen_ssa/src/back/archive.rs
+++ b/compiler/rustc_codegen_ssa/src/back/archive.rs
@@ -1,44 +1,16 @@
+use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::memmap::Mmap;
 use rustc_session::cstore::DllImport;
 use rustc_session::Session;
+use rustc_span::symbol::Symbol;
 
+use object::read::archive::ArchiveFile;
+
+use std::fmt::Display;
+use std::fs::File;
 use std::io;
 use std::path::{Path, PathBuf};
 
-pub(super) fn find_library(
-    name: &str,
-    verbatim: bool,
-    search_paths: &[PathBuf],
-    sess: &Session,
-) -> PathBuf {
-    // On Windows, static libraries sometimes show up as libfoo.a and other
-    // times show up as foo.lib
-    let oslibname = if verbatim {
-        name.to_string()
-    } else {
-        format!("{}{}{}", sess.target.staticlib_prefix, name, sess.target.staticlib_suffix)
-    };
-    let unixlibname = format!("lib{}.a", name);
-
-    for path in search_paths {
-        debug!("looking for {} inside {:?}", name, path);
-        let test = path.join(&oslibname);
-        if test.exists() {
-            return test;
-        }
-        if oslibname != unixlibname {
-            let test = path.join(&unixlibname);
-            if test.exists() {
-                return test;
-            }
-        }
-    }
-    sess.fatal(&format!(
-        "could not find native static library `{}`, \
-                         perhaps an -L flag is missing?",
-        name
-    ));
-}
-
 pub trait ArchiveBuilderBuilder {
     fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder<'a> + 'a>;
 
@@ -54,6 +26,36 @@ pub trait ArchiveBuilderBuilder {
         dll_imports: &[DllImport],
         tmpdir: &Path,
     ) -> PathBuf;
+
+    fn extract_bundled_libs(
+        &self,
+        rlib: &Path,
+        outdir: &Path,
+        bundled_lib_file_names: &FxHashSet<Symbol>,
+    ) -> Result<(), String> {
+        let message = |msg: &str, e: &dyn Display| format!("{} '{}': {}", msg, &rlib.display(), e);
+        let archive_map = unsafe {
+            Mmap::map(File::open(rlib).map_err(|e| message("failed to open file", &e))?)
+                .map_err(|e| message("failed to mmap file", &e))?
+        };
+        let archive = ArchiveFile::parse(&*archive_map)
+            .map_err(|e| message("failed to parse archive", &e))?;
+
+        for entry in archive.members() {
+            let entry = entry.map_err(|e| message("failed to read entry", &e))?;
+            let data = entry
+                .data(&*archive_map)
+                .map_err(|e| message("failed to get data from archive member", &e))?;
+            let name = std::str::from_utf8(entry.name())
+                .map_err(|e| message("failed to convert name", &e))?;
+            if !bundled_lib_file_names.contains(&Symbol::intern(name)) {
+                continue; // We need to extract only native libraries.
+            }
+            std::fs::write(&outdir.join(&name), data)
+                .map_err(|e| message("failed to write file", &e))?;
+        }
+        Ok(())
+    }
 }
 
 pub trait ArchiveBuilder<'a> {
diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs
index d2f2c7bf798..bb57fca74a2 100644
--- a/compiler/rustc_codegen_ssa/src/back/link.rs
+++ b/compiler/rustc_codegen_ssa/src/back/link.rs
@@ -1,11 +1,13 @@
 use rustc_arena::TypedArena;
 use rustc_ast::CRATE_NODE_ID;
-use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
+use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::memmap::Mmap;
 use rustc_data_structures::temp_dir::MaybeTempDir;
 use rustc_errors::{ErrorGuaranteed, Handler};
 use rustc_fs_util::fix_windows_verbatim_for_gcc;
 use rustc_hir::def_id::CrateNum;
+use rustc_metadata::find_native_static_library;
 use rustc_metadata::fs::{emit_metadata, METADATA_FILENAME};
 use rustc_middle::middle::dependency_format::Linkage;
 use rustc_middle::middle::exported_symbols::SymbolExportKind;
@@ -24,7 +26,7 @@ use rustc_target::spec::crt_objects::{CrtObjects, LinkSelfContainedDefault};
 use rustc_target::spec::{LinkOutputKind, LinkerFlavor, LldFlavor, SplitDebuginfo};
 use rustc_target::spec::{PanicStrategy, RelocModel, RelroLevel, SanitizerSet, Target};
 
-use super::archive::{find_library, ArchiveBuilder, ArchiveBuilderBuilder};
+use super::archive::{ArchiveBuilder, ArchiveBuilderBuilder};
 use super::command::Command;
 use super::linker::{self, Linker};
 use super::metadata::{create_rmeta_file, MetadataPosition};
@@ -44,7 +46,7 @@ use std::io::{BufWriter, Write};
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
 use std::process::{ExitStatus, Output, Stdio};
-use std::{ascii, char, env, fmt, fs, io, mem, str};
+use std::{env, fmt, fs, io, mem, str};
 
 pub fn ensure_removed(diag_handler: &Handler, path: &Path) {
     if let Err(e) = fs::remove_file(path) {
@@ -307,6 +309,9 @@ fn link_rlib<'a>(
         }
     }
 
+    // Used if packed_bundled_libs flag enabled.
+    let mut packed_bundled_libs = Vec::new();
+
     // Note that in this loop we are ignoring the value of `lib.cfg`. That is,
     // we may not be configured to actually include a static library if we're
     // adding it here. That's because later when we consume this rlib we'll
@@ -326,6 +331,8 @@ fn link_rlib<'a>(
     for lib in codegen_results.crate_info.used_libraries.iter() {
         match lib.kind {
             NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) }
+                if flavor == RlibFlavor::Normal && sess.opts.unstable_opts.packed_bundled_libs => {}
+            NativeLibKind::Static { bundle: None | Some(true), whole_archive: Some(true) }
                 if flavor == RlibFlavor::Normal =>
             {
                 // Don't allow mixing +bundle with +whole_archive since an rlib may contain
@@ -348,7 +355,16 @@ fn link_rlib<'a>(
         }
         if let Some(name) = lib.name {
             let location =
-                find_library(name.as_str(), lib.verbatim.unwrap_or(false), &lib_search_paths, sess);
+                find_native_static_library(name.as_str(), lib.verbatim, &lib_search_paths, sess);
+            if sess.opts.unstable_opts.packed_bundled_libs && flavor == RlibFlavor::Normal {
+                packed_bundled_libs.push(find_native_static_library(
+                    lib.filename.unwrap().as_str(),
+                    Some(true),
+                    &lib_search_paths,
+                    sess,
+                ));
+                continue;
+            }
             ab.add_archive(&location, Box::new(|_| false)).unwrap_or_else(|e| {
                 sess.fatal(&format!(
                     "failed to add native library {}: {}",
@@ -403,6 +419,12 @@ fn link_rlib<'a>(
         ab.add_file(&trailing_metadata);
     }
 
+    // Add all bundled static native library dependencies.
+    // Archives added to the end of .rlib archive, see comment above for the reason.
+    for lib in packed_bundled_libs {
+        ab.add_file(&lib)
+    }
+
     return Ok(ab);
 }
 
@@ -552,14 +574,6 @@ fn link_staticlib<'a>(
     Ok(())
 }
 
-fn escape_stdout_stderr_string(s: &[u8]) -> String {
-    str::from_utf8(s).map(|s| s.to_owned()).unwrap_or_else(|_| {
-        let mut x = "Non-UTF-8 output: ".to_string();
-        x.extend(s.iter().flat_map(|&b| ascii::escape_default(b)).map(char::from));
-        x
-    })
-}
-
 /// Use `thorin` (rust implementation of a dwarf packaging utility) to link DWARF objects into a
 /// DWARF package.
 fn link_dwarf_object<'a>(
@@ -866,7 +880,7 @@ fn link_natively<'a>(
             if !prog.status.success() {
                 let mut output = prog.stderr.clone();
                 output.extend_from_slice(&prog.stdout);
-                let escaped_output = escape_stdout_stderr_string(&output);
+                let escaped_output = escape_string(&output);
                 let mut err = sess.struct_err(&format!(
                     "linking with `{}` failed: {}",
                     linker_path.display(),
@@ -934,8 +948,8 @@ fn link_natively<'a>(
 
                 sess.abort_if_errors();
             }
-            info!("linker stderr:\n{}", escape_stdout_stderr_string(&prog.stderr));
-            info!("linker stdout:\n{}", escape_stdout_stderr_string(&prog.stdout));
+            info!("linker stderr:\n{}", escape_string(&prog.stderr));
+            info!("linker stdout:\n{}", escape_string(&prog.stdout));
         }
         Err(e) => {
             let linker_not_found = e.kind() == io::ErrorKind::NotFound;
@@ -1065,11 +1079,10 @@ fn strip_symbols_in_osx<'a>(sess: &'a Session, out_filename: &Path, option: Opti
 }
 
 fn escape_string(s: &[u8]) -> String {
-    str::from_utf8(s).map(|s| s.to_owned()).unwrap_or_else(|_| {
-        let mut x = "Non-UTF-8 output: ".to_string();
-        x.extend(s.iter().flat_map(|&b| ascii::escape_default(b)).map(char::from));
-        x
-    })
+    match str::from_utf8(s) {
+        Ok(s) => s.to_owned(),
+        Err(_) => format!("Non-UTF-8 output: {}", s.escape_ascii()),
+    }
 }
 
 fn add_sanitizer_libraries(sess: &Session, crate_type: CrateType, linker: &mut dyn Linker) {
@@ -1077,11 +1090,12 @@ fn add_sanitizer_libraries(sess: &Session, crate_type: CrateType, linker: &mut d
     // both executables and dynamic shared objects. Everywhere else the runtimes
     // are currently distributed as static libraries which should be linked to
     // executables only.
-    let needs_runtime = match crate_type {
-        CrateType::Executable => true,
-        CrateType::Dylib | CrateType::Cdylib | CrateType::ProcMacro => sess.target.is_like_osx,
-        CrateType::Rlib | CrateType::Staticlib => false,
-    };
+    let needs_runtime = !sess.target.is_like_android
+        && match crate_type {
+            CrateType::Executable => true,
+            CrateType::Dylib | CrateType::Cdylib | CrateType::ProcMacro => sess.target.is_like_osx,
+            CrateType::Rlib | CrateType::Staticlib => false,
+        };
 
     if !needs_runtime {
         return;
@@ -1173,13 +1187,6 @@ pub fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) {
             // only the linker flavor is known; use the default linker for the selected flavor
             (None, Some(flavor)) => Some((
                 PathBuf::from(match flavor {
-                    LinkerFlavor::Em => {
-                        if cfg!(windows) {
-                            "emcc.bat"
-                        } else {
-                            "emcc"
-                        }
-                    }
                     LinkerFlavor::Gcc => {
                         if cfg!(any(target_os = "solaris", target_os = "illumos")) {
                             // On historical Solaris systems, "cc" may have
@@ -1194,11 +1201,17 @@ pub fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) {
                         }
                     }
                     LinkerFlavor::Ld => "ld",
-                    LinkerFlavor::Msvc => "link.exe",
                     LinkerFlavor::Lld(_) => "lld",
-                    LinkerFlavor::PtxLinker => "rust-ptx-linker",
-                    LinkerFlavor::BpfLinker => "bpf-linker",
-                    LinkerFlavor::L4Bender => "l4-bender",
+                    LinkerFlavor::Msvc => "link.exe",
+                    LinkerFlavor::EmCc => {
+                        if cfg!(windows) {
+                            "emcc.bat"
+                        } else {
+                            "emcc"
+                        }
+                    }
+                    LinkerFlavor::Bpf => "bpf-linker",
+                    LinkerFlavor::Ptx => "rust-ptx-linker",
                 }),
                 flavor,
             )),
@@ -1208,7 +1221,7 @@ pub fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) {
                 });
 
                 let flavor = if stem == "emcc" {
-                    LinkerFlavor::Em
+                    LinkerFlavor::EmCc
                 } else if stem == "gcc"
                     || stem.ends_with("-gcc")
                     || stem == "clang"
@@ -1236,7 +1249,8 @@ pub fn linker_and_flavor(sess: &Session) -> (PathBuf, LinkerFlavor) {
 
     // linker and linker flavor specified via command line have precedence over what the target
     // specification specifies
-    if let Some(ret) = infer_from(sess, sess.opts.cg.linker.clone(), sess.opts.cg.linker_flavor) {
+    let linker_flavor = sess.opts.cg.linker_flavor.map(LinkerFlavor::from_cli);
+    if let Some(ret) = infer_from(sess, sess.opts.cg.linker.clone(), linker_flavor) {
         return ret;
     }
 
@@ -1714,6 +1728,13 @@ fn add_post_link_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor
 /// that are necessary for the linking. They are only present in symbol table but not actually
 /// used in any sections, so the linker will therefore pick relevant rlibs for linking, but
 /// unused `#[no_mangle]` or `#[used]` can still be discard by GC sections.
+///
+/// There's a few internal crates in the standard library (aka libcore and
+/// libstd) which actually have a circular dependence upon one another. This
+/// currently arises through "weak lang items" where libcore requires things
+/// like `rust_begin_unwind` but libstd ends up defining it. To get this
+/// circular dependence to work correctly we declare some of these things
+/// in this synthetic object.
 fn add_linked_symbol_object(
     cmd: &mut dyn Linker,
     sess: &Session,
@@ -2113,11 +2134,11 @@ fn add_order_independent_options(
         });
     }
 
-    if flavor == LinkerFlavor::PtxLinker {
+    if flavor == LinkerFlavor::Ptx {
         // Provide the linker with fallback to internal `target-cpu`.
         cmd.arg("--fallback-arch");
         cmd.arg(&codegen_results.crate_info.target_cpu);
-    } else if flavor == LinkerFlavor::BpfLinker {
+    } else if flavor == LinkerFlavor::Bpf {
         cmd.arg("--cpu");
         cmd.arg(&codegen_results.crate_info.target_cpu);
         cmd.arg("--cpu-features");
@@ -2333,72 +2354,25 @@ fn add_upstream_rust_crates<'a>(
     // crates.
     let deps = &codegen_results.crate_info.used_crates;
 
-    // There's a few internal crates in the standard library (aka libcore and
-    // libstd) which actually have a circular dependence upon one another. This
-    // currently arises through "weak lang items" where libcore requires things
-    // like `rust_begin_unwind` but libstd ends up defining it. To get this
-    // circular dependence to work correctly in all situations we'll need to be
-    // sure to correctly apply the `--start-group` and `--end-group` options to
-    // GNU linkers, otherwise if we don't use any other symbol from the standard
-    // library it'll get discarded and the whole application won't link.
-    //
-    // In this loop we're calculating the `group_end`, after which crate to
-    // pass `--end-group` and `group_start`, before which crate to pass
-    // `--start-group`. We currently do this by passing `--end-group` after
-    // the first crate (when iterating backwards) that requires a lang item
-    // defined somewhere else. Once that's set then when we've defined all the
-    // necessary lang items we'll pass `--start-group`.
-    //
-    // Note that this isn't amazing logic for now but it should do the trick
-    // for the current implementation of the standard library.
-    let mut group_end = None;
-    let mut group_start = None;
-    // Crates available for linking thus far.
-    let mut available = FxHashSet::default();
-    // Crates required to satisfy dependencies discovered so far.
-    let mut required = FxHashSet::default();
-
-    let info = &codegen_results.crate_info;
-    for &cnum in deps.iter().rev() {
-        if let Some(missing) = info.missing_lang_items.get(&cnum) {
-            let missing_crates = missing.iter().map(|i| info.lang_item_to_crate.get(i).copied());
-            required.extend(missing_crates);
-        }
-
-        required.insert(Some(cnum));
-        available.insert(Some(cnum));
-
-        if required.len() > available.len() && group_end.is_none() {
-            group_end = Some(cnum);
-        }
-        if required.len() == available.len() && group_end.is_some() {
-            group_start = Some(cnum);
-            break;
-        }
-    }
-
-    // If we didn't end up filling in all lang items from upstream crates then
-    // we'll be filling it in with our crate. This probably means we're the
-    // standard library itself, so skip this for now.
-    if group_end.is_some() && group_start.is_none() {
-        group_end = None;
-    }
-
     let mut compiler_builtins = None;
     let search_path = OnceCell::new();
 
     for &cnum in deps.iter() {
-        if group_start == Some(cnum) {
-            cmd.group_start();
-        }
-
         // We may not pass all crates through to the linker. Some crates may
         // appear statically in an existing dylib, meaning we'll pick up all the
         // symbols from the dylib.
         let src = &codegen_results.crate_info.used_crate_source[&cnum];
         match data[cnum.as_usize() - 1] {
             _ if codegen_results.crate_info.profiler_runtime == Some(cnum) => {
-                add_static_crate(cmd, sess, archive_builder_builder, codegen_results, tmpdir, cnum);
+                add_static_crate(
+                    cmd,
+                    sess,
+                    archive_builder_builder,
+                    codegen_results,
+                    tmpdir,
+                    cnum,
+                    &Default::default(),
+                );
             }
             // compiler-builtins are always placed last to ensure that they're
             // linked correctly.
@@ -2408,7 +2382,23 @@ fn add_upstream_rust_crates<'a>(
             }
             Linkage::NotLinked | Linkage::IncludedFromDylib => {}
             Linkage::Static => {
-                add_static_crate(cmd, sess, archive_builder_builder, codegen_results, tmpdir, cnum);
+                let bundled_libs = if sess.opts.unstable_opts.packed_bundled_libs {
+                    codegen_results.crate_info.native_libraries[&cnum]
+                        .iter()
+                        .filter_map(|lib| lib.filename)
+                        .collect::<FxHashSet<_>>()
+                } else {
+                    Default::default()
+                };
+                add_static_crate(
+                    cmd,
+                    sess,
+                    archive_builder_builder,
+                    codegen_results,
+                    tmpdir,
+                    cnum,
+                    &bundled_libs,
+                );
 
                 // Link static native libs with "-bundle" modifier only if the crate they originate from
                 // is being linked statically to the current crate.  If it's linked dynamically
@@ -2419,6 +2409,14 @@ fn add_upstream_rust_crates<'a>(
                 // external build system already has the native dependencies defined, and it
                 // will provide them to the linker itself.
                 if sess.opts.unstable_opts.link_native_libraries {
+                    if sess.opts.unstable_opts.packed_bundled_libs {
+                        // If rlib contains native libs as archives, unpack them to tmpdir.
+                        let rlib = &src.rlib.as_ref().unwrap().0;
+                        archive_builder_builder
+                            .extract_bundled_libs(rlib, tmpdir, &bundled_libs)
+                            .unwrap_or_else(|e| sess.fatal(e));
+                    }
+
                     let mut last = (None, NativeLibKind::Unspecified, None);
                     for lib in &codegen_results.crate_info.native_libraries[&cnum] {
                         let Some(name) = lib.name else {
@@ -2451,6 +2449,14 @@ fn add_upstream_rust_crates<'a>(
                                 bundle: Some(false),
                                 whole_archive: Some(false) | None,
                             } => {
+                                // HACK/FIXME: Fixup a circular dependency between libgcc and libc
+                                // with glibc. This logic should be moved to the libc crate.
+                                if sess.target.os == "linux"
+                                    && sess.target.env == "gnu"
+                                    && name == "c"
+                                {
+                                    cmd.link_staticlib("gcc", false);
+                                }
                                 cmd.link_staticlib(name, lib.verbatim.unwrap_or(false));
                             }
                             NativeLibKind::LinkArg => {
@@ -2460,20 +2466,23 @@ fn add_upstream_rust_crates<'a>(
                             | NativeLibKind::Framework { .. }
                             | NativeLibKind::Unspecified
                             | NativeLibKind::RawDylib => {}
-                            NativeLibKind::Static {
-                                bundle: Some(true) | None,
-                                whole_archive: _,
-                            } => {}
+                            NativeLibKind::Static { bundle: Some(true) | None, whole_archive } => {
+                                if sess.opts.unstable_opts.packed_bundled_libs {
+                                    // If rlib contains native libs as archives, they are unpacked to tmpdir.
+                                    let path = tmpdir.join(lib.filename.unwrap().as_str());
+                                    if whole_archive == Some(true) {
+                                        cmd.link_whole_rlib(&path);
+                                    } else {
+                                        cmd.link_rlib(&path);
+                                    }
+                                }
+                            }
                         }
                     }
                 }
             }
             Linkage::Dynamic => add_dynamic_crate(cmd, sess, &src.dylib.as_ref().unwrap().0),
         }
-
-        if group_end == Some(cnum) {
-            cmd.group_end();
-        }
     }
 
     // compiler-builtins are always placed last to ensure that they're
@@ -2482,7 +2491,15 @@ fn add_upstream_rust_crates<'a>(
     // was already "included" in a dylib (e.g., `libstd` when `-C prefer-dynamic`
     // is used)
     if let Some(cnum) = compiler_builtins {
-        add_static_crate(cmd, sess, archive_builder_builder, codegen_results, tmpdir, cnum);
+        add_static_crate(
+            cmd,
+            sess,
+            archive_builder_builder,
+            codegen_results,
+            tmpdir,
+            cnum,
+            &Default::default(),
+        );
     }
 
     // Converts a library file-stem into a cc -l argument
@@ -2515,6 +2532,7 @@ fn add_upstream_rust_crates<'a>(
         codegen_results: &CodegenResults,
         tmpdir: &Path,
         cnum: CrateNum,
+        bundled_lib_file_names: &FxHashSet<Symbol>,
     ) {
         let src = &codegen_results.crate_info.used_crate_source[&cnum];
         let cratepath = &src.rlib.as_ref().unwrap().0;
@@ -2543,6 +2561,7 @@ fn add_upstream_rust_crates<'a>(
         let dst = tmpdir.join(cratepath.file_name().unwrap());
         let name = cratepath.file_name().unwrap().to_str().unwrap();
         let name = &name[3..name.len() - 5]; // chop off lib/.rlib
+        let bundled_lib_file_names = bundled_lib_file_names.clone();
 
         sess.prof.generic_activity_with_arg("link_altering_rlib", name).run(|| {
             let canonical_name = name.replace('-', "_");
@@ -2576,6 +2595,15 @@ fn add_upstream_rust_crates<'a>(
                     let skip_because_lto =
                         upstream_rust_objects_already_included && is_rust_object && is_builtins;
 
+                    // We skip native libraries because:
+                    // 1. This native libraries won't be used from the generated rlib,
+                    //    so we can throw them away to avoid the copying work.
+                    // 2. We can't allow it to be a single remaining entry in archive
+                    //    as some linkers may complain on that.
+                    if bundled_lib_file_names.contains(&Symbol::intern(f)) {
+                        return true;
+                    }
+
                     if skip_because_cfg_say_so || skip_because_lto {
                         return true;
                     }
@@ -2797,20 +2825,24 @@ fn add_gcc_ld_path(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) {
         if let LinkerFlavor::Gcc = flavor {
             match ld_impl {
                 LdImpl::Lld => {
-                    let tools_path = sess.get_tools_search_paths(false);
-                    let gcc_ld_dir = tools_path
-                        .into_iter()
-                        .map(|p| p.join("gcc-ld"))
-                        .find(|p| {
-                            p.join(if sess.host.is_like_windows { "ld.exe" } else { "ld" }).exists()
-                        })
-                        .unwrap_or_else(|| sess.fatal("rust-lld (as ld) not found"));
-                    cmd.arg({
-                        let mut arg = OsString::from("-B");
-                        arg.push(gcc_ld_dir);
-                        arg
-                    });
-                    cmd.arg(format!("-Wl,-rustc-lld-flavor={}", sess.target.lld_flavor.as_str()));
+                    // Implement the "self-contained" part of -Zgcc-ld
+                    // by adding rustc distribution directories to the tool search path.
+                    for path in sess.get_tools_search_paths(false) {
+                        cmd.arg({
+                            let mut arg = OsString::from("-B");
+                            arg.push(path.join("gcc-ld"));
+                            arg
+                        });
+                    }
+                    // Implement the "linker flavor" part of -Zgcc-ld
+                    // by asking cc to use some kind of lld.
+                    cmd.arg("-fuse-ld=lld");
+                    if sess.target.lld_flavor != LldFlavor::Ld {
+                        // Tell clang to use a non-default LLD flavor.
+                        // Gcc doesn't understand the target option, but we currently assume
+                        // that gcc is not used for Apple and Wasm targets (#97402).
+                        cmd.arg(format!("--target={}", sess.target.llvm_target));
+                    }
                 }
             }
         } else {
diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs
index ce51b2e9531..e0bd7a33f73 100644
--- a/compiler/rustc_codegen_ssa/src/back/linker.rs
+++ b/compiler/rustc_codegen_ssa/src/back/linker.rs
@@ -1,4 +1,3 @@
-use super::archive;
 use super::command::Command;
 use super::symbol_export;
 use rustc_span::symbol::sym;
@@ -11,6 +10,7 @@ use std::path::{Path, PathBuf};
 use std::{env, mem, str};
 
 use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
+use rustc_metadata::find_native_static_library;
 use rustc_middle::middle::dependency_format::Linkage;
 use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo, SymbolExportKind};
 use rustc_middle::ty::TyCtxt;
@@ -126,29 +126,26 @@ pub fn get_linker<'a>(
     // to the linker args construction.
     assert!(cmd.get_args().is_empty() || sess.target.vendor == "uwp");
     match flavor {
-        LinkerFlavor::Lld(LldFlavor::Link) | LinkerFlavor::Msvc => {
-            Box::new(MsvcLinker { cmd, sess }) as Box<dyn Linker>
-        }
-        LinkerFlavor::Em => Box::new(EmLinker { cmd, sess }) as Box<dyn Linker>,
         LinkerFlavor::Gcc => {
             Box::new(GccLinker { cmd, sess, target_cpu, hinted_static: false, is_ld: false })
                 as Box<dyn Linker>
         }
-
+        LinkerFlavor::Ld if sess.target.os == "l4re" => {
+            Box::new(L4Bender::new(cmd, sess)) as Box<dyn Linker>
+        }
         LinkerFlavor::Lld(LldFlavor::Ld)
         | LinkerFlavor::Lld(LldFlavor::Ld64)
         | LinkerFlavor::Ld => {
             Box::new(GccLinker { cmd, sess, target_cpu, hinted_static: false, is_ld: true })
                 as Box<dyn Linker>
         }
-
+        LinkerFlavor::Lld(LldFlavor::Link) | LinkerFlavor::Msvc => {
+            Box::new(MsvcLinker { cmd, sess }) as Box<dyn Linker>
+        }
         LinkerFlavor::Lld(LldFlavor::Wasm) => Box::new(WasmLd::new(cmd, sess)) as Box<dyn Linker>,
-
-        LinkerFlavor::PtxLinker => Box::new(PtxLinker { cmd, sess }) as Box<dyn Linker>,
-
-        LinkerFlavor::BpfLinker => Box::new(BpfLinker { cmd, sess }) as Box<dyn Linker>,
-
-        LinkerFlavor::L4Bender => Box::new(L4Bender::new(cmd, sess)) as Box<dyn Linker>,
+        LinkerFlavor::EmCc => Box::new(EmLinker { cmd, sess }) as Box<dyn Linker>,
+        LinkerFlavor::Bpf => Box::new(BpfLinker { cmd, sess }) as Box<dyn Linker>,
+        LinkerFlavor::Ptx => Box::new(PtxLinker { cmd, sess }) as Box<dyn Linker>,
     }
 }
 
@@ -186,8 +183,6 @@ pub trait Linker {
     fn no_default_libraries(&mut self);
     fn export_symbols(&mut self, tmpdir: &Path, crate_type: CrateType, symbols: &[String]);
     fn subsystem(&mut self, subsystem: &str);
-    fn group_start(&mut self);
-    fn group_end(&mut self);
     fn linker_plugin_lto(&mut self);
     fn add_eh_frame_header(&mut self) {}
     fn add_no_exec(&mut self) {}
@@ -444,7 +439,10 @@ impl<'a> Linker for GccLinker<'a> {
             }
         }
         self.hint_dynamic();
-        self.cmd.arg(format!("-l{}{}", if verbatim { ":" } else { "" }, lib));
+        self.cmd.arg(format!(
+            "-l{}{lib}",
+            if verbatim && self.sess.target.linker_is_gnu { ":" } else { "" },
+        ));
         if !as_needed {
             if self.sess.target.is_like_osx {
                 // See above FIXME comment
@@ -455,7 +453,10 @@ impl<'a> Linker for GccLinker<'a> {
     }
     fn link_staticlib(&mut self, lib: &str, verbatim: bool) {
         self.hint_static();
-        self.cmd.arg(format!("-l{}{}", if verbatim { ":" } else { "" }, lib));
+        self.cmd.arg(format!(
+            "-l{}{lib}",
+            if verbatim && self.sess.target.linker_is_gnu { ":" } else { "" },
+        ));
     }
     fn link_rlib(&mut self, lib: &Path) {
         self.hint_static();
@@ -509,17 +510,17 @@ impl<'a> Linker for GccLinker<'a> {
         self.hint_static();
         let target = &self.sess.target;
         if !target.is_like_osx {
-            self.linker_arg("--whole-archive").cmd.arg(format!(
-                "-l{}{}",
-                if verbatim { ":" } else { "" },
-                lib
+            self.linker_arg("--whole-archive");
+            self.cmd.arg(format!(
+                "-l{}{lib}",
+                if verbatim && self.sess.target.linker_is_gnu { ":" } else { "" },
             ));
             self.linker_arg("--no-whole-archive");
         } else {
             // -force_load is the macOS equivalent of --whole-archive, but it
             // involves passing the full path to the library to link.
             self.linker_arg("-force_load");
-            let lib = archive::find_library(lib, verbatim, search_path, &self.sess);
+            let lib = find_native_static_library(lib, Some(verbatim), search_path, &self.sess);
             self.linker_arg(&lib);
         }
     }
@@ -733,18 +734,6 @@ impl<'a> Linker for GccLinker<'a> {
         self.hint_dynamic(); // Reset to default before returning the composed command line.
     }
 
-    fn group_start(&mut self) {
-        if self.takes_hints() {
-            self.linker_arg("--start-group");
-        }
-    }
-
-    fn group_end(&mut self) {
-        if self.takes_hints() {
-            self.linker_arg("--end-group");
-        }
-    }
-
     fn linker_plugin_lto(&mut self) {
         match self.sess.opts.cg.linker_plugin_lto {
             LinkerPluginLto::Disabled => {
@@ -1022,10 +1011,6 @@ impl<'a> Linker for MsvcLinker<'a> {
         }
     }
 
-    // MSVC doesn't need group indicators
-    fn group_start(&mut self) {}
-    fn group_end(&mut self) {}
-
     fn linker_plugin_lto(&mut self) {
         // Do nothing
     }
@@ -1168,10 +1153,6 @@ impl<'a> Linker for EmLinker<'a> {
         // noop
     }
 
-    // Appears not necessary on Emscripten
-    fn group_start(&mut self) {}
-    fn group_end(&mut self) {}
-
     fn linker_plugin_lto(&mut self) {
         // Do nothing
     }
@@ -1347,10 +1328,6 @@ impl<'a> Linker for WasmLd<'a> {
 
     fn subsystem(&mut self, _subsystem: &str) {}
 
-    // Not needed for now with LLD
-    fn group_start(&mut self) {}
-    fn group_end(&mut self) {}
-
     fn linker_plugin_lto(&mut self) {
         // Do nothing for now
     }
@@ -1479,14 +1456,6 @@ impl<'a> Linker for L4Bender<'a> {
         self.hint_static(); // Reset to default before returning the composed command line.
     }
 
-    fn group_start(&mut self) {
-        self.cmd.arg("--start-group");
-    }
-
-    fn group_end(&mut self) {
-        self.cmd.arg("--end-group");
-    }
-
     fn linker_plugin_lto(&mut self) {}
 
     fn control_flow_guard(&mut self) {}
@@ -1667,10 +1636,6 @@ impl<'a> Linker for PtxLinker<'a> {
 
     fn subsystem(&mut self, _subsystem: &str) {}
 
-    fn group_start(&mut self) {}
-
-    fn group_end(&mut self) {}
-
     fn linker_plugin_lto(&mut self) {}
 }
 
@@ -1780,9 +1745,5 @@ impl<'a> Linker for BpfLinker<'a> {
 
     fn subsystem(&mut self, _subsystem: &str) {}
 
-    fn group_start(&mut self) {}
-
-    fn group_end(&mut self) {}
-
     fn linker_plugin_lto(&mut self) {}
 }
diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
index 32b340832ce..8d7e2c5cf39 100644
--- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
+++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
@@ -540,7 +540,7 @@ pub fn linking_symbol_name_for_instance_in_crate<'tcx>(
         .map(|fnabi| (fnabi.conv, &fnabi.args[..]))
         .unwrap_or((Conv::Rust, &[]));
 
-    // Decorate symbols with prefices, suffices and total number of bytes of arguments.
+    // Decorate symbols with prefixes, suffixes and total number of bytes of arguments.
     // Reference: https://docs.microsoft.com/en-us/cpp/build/reference/decorated-names?view=msvc-170
     let (prefix, suffix) = match conv {
         Conv::X86Fastcall => ("@", "@"),
diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs
index 68f3b19b715..680b9b642d9 100644
--- a/compiler/rustc_codegen_ssa/src/back/write.rs
+++ b/compiler/rustc_codegen_ssa/src/back/write.rs
@@ -113,7 +113,6 @@ pub struct ModuleConfig {
     pub vectorize_slp: bool,
     pub merge_functions: bool,
     pub inline_threshold: Option<u32>,
-    pub new_llvm_pass_manager: Option<bool>,
     pub emit_lifetime_markers: bool,
     pub llvm_plugins: Vec<String>,
 }
@@ -265,7 +264,6 @@ impl ModuleConfig {
             },
 
             inline_threshold: sess.opts.cg.inline_threshold,
-            new_llvm_pass_manager: sess.opts.unstable_opts.new_llvm_pass_manager,
             emit_lifetime_markers: sess.emit_lifetime_markers(),
             llvm_plugins: if_regular!(sess.opts.unstable_opts.llvm_plugins.clone(), vec![]),
         }
diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs
index 4c6be3f9108..35fd86c1735 100644
--- a/compiler/rustc_codegen_ssa/src/base.rs
+++ b/compiler/rustc_codegen_ssa/src/base.rs
@@ -12,7 +12,7 @@ use crate::traits::*;
 use crate::{CachedModuleCodegen, CompiledModule, CrateInfo, MemFlags, ModuleCodegen, ModuleKind};
 
 use rustc_attr as attr;
-use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::profiling::{get_resident_set_size, print_time_passes_entry};
 
 use rustc_data_structures::sync::par_iter;
@@ -21,10 +21,12 @@ use rustc_data_structures::sync::ParallelIterator;
 use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LOCAL_CRATE};
 use rustc_hir::lang_items::LangItem;
+use rustc_hir::weak_lang_items::WEAK_ITEMS_SYMBOLS;
 use rustc_index::vec::Idx;
 use rustc_metadata::EncodedMetadata;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
 use rustc_middle::middle::exported_symbols;
+use rustc_middle::middle::exported_symbols::SymbolExportKind;
 use rustc_middle::middle::lang_items;
 use rustc_middle::mir::mono::{CodegenUnit, CodegenUnitNameBuilder, MonoItem};
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout};
@@ -34,6 +36,7 @@ use rustc_session::cgu_reuse_tracker::CguReuse;
 use rustc_session::config::{self, CrateType, EntryFnType, OutputType};
 use rustc_session::Session;
 use rustc_span::symbol::sym;
+use rustc_span::Symbol;
 use rustc_span::{DebuggerVisualizerFile, DebuggerVisualizerType};
 use rustc_target::abi::{Align, VariantIdx};
 
@@ -163,6 +166,11 @@ pub fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
             if let Some(entry_idx) = vptr_entry_idx {
                 let ptr_ty = cx.type_i8p();
                 let ptr_align = cx.tcx().data_layout.pointer_align.abi;
+                let vtable_ptr_ty = cx.scalar_pair_element_backend_type(
+                    cx.layout_of(cx.tcx().mk_mut_ptr(target)),
+                    1,
+                    true,
+                );
                 let llvtable = bx.pointercast(old_info, bx.type_ptr_to(ptr_ty));
                 let gep = bx.inbounds_gep(
                     ptr_ty,
@@ -173,7 +181,7 @@ pub fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                 bx.nonnull_metadata(new_vptr);
                 // VTable loads are invariant.
                 bx.set_invariant_load(new_vptr);
-                new_vptr
+                bx.pointercast(new_vptr, vtable_ptr_ty)
             } else {
                 old_info
             }
@@ -389,15 +397,14 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 
     let main_llfn = cx.get_fn_addr(instance);
 
-    let use_start_lang_item = EntryFnType::Start != entry_type;
-    let entry_fn = create_entry_fn::<Bx>(cx, main_llfn, main_def_id, use_start_lang_item);
+    let entry_fn = create_entry_fn::<Bx>(cx, main_llfn, main_def_id, entry_type);
     return Some(entry_fn);
 
     fn create_entry_fn<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         cx: &'a Bx::CodegenCx,
         rust_main: Bx::Value,
         rust_main_def_id: DefId,
-        use_start_lang_item: bool,
+        entry_type: EntryFnType,
     ) -> Bx::Function {
         // The entry function is either `int main(void)` or `int main(int argc, char **argv)`,
         // depending on whether the target needs `argc` and `argv` to be passed in.
@@ -442,7 +449,7 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         let i8pp_ty = cx.type_ptr_to(cx.type_i8p());
         let (arg_argc, arg_argv) = get_argc_argv(cx, &mut bx);
 
-        let (start_fn, start_ty, args) = if use_start_lang_item {
+        let (start_fn, start_ty, args) = if let EntryFnType::Main { sigpipe } = entry_type {
             let start_def_id = cx.tcx().require_lang_item(LangItem::Start, None);
             let start_fn = cx.get_fn_addr(
                 ty::Instance::resolve(
@@ -454,8 +461,13 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                 .unwrap()
                 .unwrap(),
             );
-            let start_ty = cx.type_func(&[cx.val_ty(rust_main), isize_ty, i8pp_ty], isize_ty);
-            (start_fn, start_ty, vec![rust_main, arg_argc, arg_argv])
+
+            let i8_ty = cx.type_i8();
+            let arg_sigpipe = bx.const_u8(sigpipe);
+
+            let start_ty =
+                cx.type_func(&[cx.val_ty(rust_main), isize_ty, i8pp_ty, i8_ty], isize_ty);
+            (start_fn, start_ty, vec![rust_main, arg_argc, arg_argv, arg_sigpipe])
         } else {
             debug!("using user-defined start fn");
             let start_ty = cx.type_func(&[isize_ty, i8pp_ty], isize_ty);
@@ -811,21 +823,16 @@ impl CrateInfo {
             crate_name: Default::default(),
             used_crates,
             used_crate_source: Default::default(),
-            lang_item_to_crate: Default::default(),
-            missing_lang_items: Default::default(),
             dependency_formats: tcx.dependency_formats(()).clone(),
             windows_subsystem,
             natvis_debugger_visualizers: Default::default(),
         };
-        let lang_items = tcx.lang_items();
-
         let crates = tcx.crates(());
 
         let n_crates = crates.len();
         info.native_libraries.reserve(n_crates);
         info.crate_name.reserve(n_crates);
         info.used_crate_source.reserve(n_crates);
-        info.missing_lang_items.reserve(n_crates);
 
         for &cnum in crates.iter() {
             info.native_libraries
@@ -843,17 +850,37 @@ impl CrateInfo {
             if tcx.is_no_builtins(cnum) {
                 info.is_no_builtins.insert(cnum);
             }
-            let missing = tcx.missing_lang_items(cnum);
-            for &item in missing.iter() {
-                if let Ok(id) = lang_items.require(item) {
-                    info.lang_item_to_crate.insert(item, id.krate);
-                }
-            }
+        }
 
-            // No need to look for lang items that don't actually need to exist.
-            let missing =
-                missing.iter().cloned().filter(|&l| lang_items::required(tcx, l)).collect();
-            info.missing_lang_items.insert(cnum, missing);
+        // Handle circular dependencies in the standard library.
+        // See comment before `add_linked_symbol_object` function for the details.
+        // With msvc-like linkers it's both unnecessary (they support circular dependencies),
+        // and causes linking issues (when weak lang item symbols are "privatized" by LTO).
+        let target = &tcx.sess.target;
+        if !target.is_like_msvc {
+            let missing_weak_lang_items: FxHashSet<&Symbol> = info
+                .used_crates
+                .iter()
+                .flat_map(|cnum| {
+                    tcx.missing_lang_items(*cnum)
+                        .iter()
+                        .filter(|l| lang_items::required(tcx, **l))
+                        .filter_map(|item| WEAK_ITEMS_SYMBOLS.get(item))
+                })
+                .collect();
+            let prefix = if target.is_like_windows && target.arch == "x86" { "_" } else { "" };
+            info.linked_symbols
+                .iter_mut()
+                .filter(|(crate_type, _)| {
+                    !matches!(crate_type, CrateType::Rlib | CrateType::Staticlib)
+                })
+                .for_each(|(_, linked_symbols)| {
+                    linked_symbols.extend(
+                        missing_weak_lang_items
+                            .iter()
+                            .map(|item| (format!("{prefix}{item}"), SymbolExportKind::Text)),
+                    )
+                });
         }
 
         let embed_visualizers = tcx.sess.crate_types().iter().any(|&crate_type| match crate_type {
@@ -874,7 +901,7 @@ impl CrateInfo {
             }
         });
 
-        if tcx.sess.target.is_like_msvc && embed_visualizers {
+        if target.is_like_msvc && embed_visualizers {
             info.natvis_debugger_visualizers =
                 collect_debugger_visualizers_transitive(tcx, DebuggerVisualizerType::Natvis);
         }
diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs
index 0faf51b062b..3ef9a634e18 100644
--- a/compiler/rustc_codegen_ssa/src/lib.rs
+++ b/compiler/rustc_codegen_ssa/src/lib.rs
@@ -1,7 +1,6 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![feature(box_patterns)]
 #![feature(try_blocks)]
-#![feature(let_else)]
 #![feature(once_cell)]
 #![feature(associated_type_bounds)]
 #![feature(strict_provenance)]
@@ -25,7 +24,6 @@ use rustc_ast as ast;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::sync::Lrc;
 use rustc_hir::def_id::CrateNum;
-use rustc_hir::LangItem;
 use rustc_middle::dep_graph::WorkProduct;
 use rustc_middle::middle::dependency_format::Dependencies;
 use rustc_middle::middle::exported_symbols::SymbolExportKind;
@@ -113,6 +111,7 @@ bitflags::bitflags! {
 pub struct NativeLib {
     pub kind: NativeLibKind,
     pub name: Option<Symbol>,
+    pub filename: Option<Symbol>,
     pub cfg: Option<ast::MetaItem>,
     pub verbatim: Option<bool>,
     pub dll_imports: Vec<cstore::DllImport>,
@@ -122,6 +121,7 @@ impl From<&cstore::NativeLib> for NativeLib {
     fn from(lib: &cstore::NativeLib) -> Self {
         NativeLib {
             kind: lib.kind,
+            filename: lib.filename,
             name: lib.name,
             cfg: lib.cfg.clone(),
             verbatim: lib.verbatim,
@@ -152,8 +152,6 @@ pub struct CrateInfo {
     pub used_libraries: Vec<NativeLib>,
     pub used_crate_source: FxHashMap<CrateNum, Lrc<CrateSource>>,
     pub used_crates: Vec<CrateNum>,
-    pub lang_item_to_crate: FxHashMap<LangItem, CrateNum>,
-    pub missing_lang_items: FxHashMap<CrateNum, Vec<LangItem>>,
     pub dependency_formats: Lrc<Dependencies>,
     pub windows_subsystem: Option<String>,
     pub natvis_debugger_visualizers: BTreeSet<DebuggerVisualizerFile>,
diff --git a/compiler/rustc_codegen_ssa/src/meth.rs b/compiler/rustc_codegen_ssa/src/meth.rs
index f8e982b7751..cae46ebd2e9 100644
--- a/compiler/rustc_codegen_ssa/src/meth.rs
+++ b/compiler/rustc_codegen_ssa/src/meth.rs
@@ -69,7 +69,7 @@ impl<'a, 'tcx> VirtualIndex {
 fn expect_dyn_trait_in_self<'tcx>(ty: Ty<'tcx>) -> ty::PolyExistentialTraitRef<'tcx> {
     for arg in ty.peel_refs().walk() {
         if let GenericArgKind::Type(ty) = arg.unpack() {
-            if let ty::Dynamic(data, _) = ty.kind() {
+            if let ty::Dynamic(data, _, _) = ty.kind() {
                 return data.principal().expect("expected principal trait object");
             }
         }
@@ -86,6 +86,7 @@ fn expect_dyn_trait_in_self<'tcx>(ty: Ty<'tcx>) -> ty::PolyExistentialTraitRef<'
 /// The `trait_ref` encodes the erased self type. Hence if we are
 /// making an object `Foo<dyn Trait>` from a value of type `Foo<T>`, then
 /// `trait_ref` would map `T: Trait`.
+#[instrument(level = "debug", skip(cx))]
 pub fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>(
     cx: &Cx,
     ty: Ty<'tcx>,
@@ -93,8 +94,6 @@ pub fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>(
 ) -> Cx::Value {
     let tcx = cx.tcx();
 
-    debug!("get_vtable(ty={:?}, trait_ref={:?})", ty, trait_ref);
-
     // Check the cache.
     if let Some(&val) = cx.vtables().borrow().get(&(ty, trait_ref)) {
         return val;
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index 5c67d3b6431..a6b226ef720 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -13,8 +13,7 @@ use rustc_ast as ast;
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_hir::lang_items::LangItem;
 use rustc_index::vec::Idx;
-use rustc_middle::mir::AssertKind;
-use rustc_middle::mir::{self, SwitchTargets};
+use rustc_middle::mir::{self, AssertKind, SwitchTargets};
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
 use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
 use rustc_middle::ty::{self, Instance, Ty, TypeVisitable};
@@ -368,6 +367,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         bx.ret(llval);
     }
 
+    #[tracing::instrument(level = "trace", skip(self, helper, bx))]
     fn codegen_drop_terminator(
         &mut self,
         helper: TerminatorCodegenHelper<'tcx>,
@@ -398,14 +398,75 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let (drop_fn, fn_abi) = match ty.kind() {
             // FIXME(eddyb) perhaps move some of this logic into
             // `Instance::resolve_drop_in_place`?
-            ty::Dynamic(..) => {
+            ty::Dynamic(_, _, ty::Dyn) => {
+                // IN THIS ARM, WE HAVE:
+                // ty = *mut (dyn Trait)
+                // which is: exists<T> ( *mut T,    Vtable<T: Trait> )
+                //                       args[0]    args[1]
+                //
+                // args = ( Data, Vtable )
+                //                  |
+                //                  v
+                //                /-------\
+                //                | ...   |
+                //                \-------/
+                //
                 let virtual_drop = Instance {
                     def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0),
                     substs: drop_fn.substs,
                 };
+                debug!("ty = {:?}", ty);
+                debug!("drop_fn = {:?}", drop_fn);
+                debug!("args = {:?}", args);
                 let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
                 let vtable = args[1];
+                // Truncate vtable off of args list
+                args = &args[..1];
+                (
+                    meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
+                        .get_fn(&mut bx, vtable, ty, &fn_abi),
+                    fn_abi,
+                )
+            }
+            ty::Dynamic(_, _, ty::DynStar) => {
+                // IN THIS ARM, WE HAVE:
+                // ty = *mut (dyn* Trait)
+                // which is: *mut exists<T: sizeof(T) == sizeof(usize)> (T, Vtable<T: Trait>)
+                //
+                // args = [ * ]
+                //          |
+                //          v
+                //      ( Data, Vtable )
+                //                |
+                //                v
+                //              /-------\
+                //              | ...   |
+                //              \-------/
+                //
+                //
+                // WE CAN CONVERT THIS INTO THE ABOVE LOGIC BY DOING
+                //
+                // data = &(*args[0]).0    // gives a pointer to Data above (really the same pointer)
+                // vtable = (*args[0]).1   // loads the vtable out
+                // (data, vtable)          // an equivalent Rust `*mut dyn Trait`
+                //
+                // SO THEN WE CAN USE THE ABOVE CODE.
+                let virtual_drop = Instance {
+                    def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0),
+                    substs: drop_fn.substs,
+                };
+                debug!("ty = {:?}", ty);
+                debug!("drop_fn = {:?}", drop_fn);
+                debug!("args = {:?}", args);
+                let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
+                let data = args[0];
+                let data_ty = bx.cx().backend_type(place.layout);
+                let vtable_ptr =
+                    bx.gep(data_ty, data, &[bx.cx().const_i32(0), bx.cx().const_i32(1)]);
+                let vtable = bx.load(bx.type_i8p(), vtable_ptr, abi::Align::ONE);
+                // Truncate vtable off of args list
                 args = &args[..1];
+                debug!("args' = {:?}", args);
                 (
                     meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
                         .get_fn(&mut bx, vtable, ty, &fn_abi),
@@ -846,7 +907,30 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         llargs.push(data_ptr);
                         continue;
                     }
-                    _ => span_bug!(span, "can't codegen a virtual call on {:?}", op),
+                    Immediate(_) => {
+                        let ty::Ref(_, ty, _) = op.layout.ty.kind() else {
+                            span_bug!(span, "can't codegen a virtual call on {:#?}", op);
+                        };
+                        if !ty.is_dyn_star() {
+                            span_bug!(span, "can't codegen a virtual call on {:#?}", op);
+                        }
+                        // FIXME(dyn-star): Make sure this is done on a &dyn* receiver
+                        let place = op.deref(bx.cx());
+                        let data_ptr = place.project_field(&mut bx, 0);
+                        let meta_ptr = place.project_field(&mut bx, 1);
+                        let meta = bx.load_operand(meta_ptr);
+                        llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(
+                            &mut bx,
+                            meta.immediate(),
+                            op.layout.ty,
+                            &fn_abi,
+                        ));
+                        llargs.push(data_ptr.llval);
+                        continue;
+                    }
+                    _ => {
+                        span_bug!(span, "can't codegen a virtual call on {:#?}", op);
+                    }
                 }
             }
 
diff --git a/compiler/rustc_codegen_ssa/src/mir/constant.rs b/compiler/rustc_codegen_ssa/src/mir/constant.rs
index 9a995fbf65c..4c6ab457c49 100644
--- a/compiler/rustc_codegen_ssa/src/mir/constant.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/constant.rs
@@ -25,26 +25,26 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         constant: &mir::Constant<'tcx>,
     ) -> Result<ConstValue<'tcx>, ErrorHandled> {
         let ct = self.monomorphize(constant.literal);
-        let ct = match ct {
-            mir::ConstantKind::Ty(ct) => ct,
+        let uv = match ct {
+            mir::ConstantKind::Ty(ct) => match ct.kind() {
+                ty::ConstKind::Unevaluated(uv) => uv.expand(),
+                ty::ConstKind::Value(val) => {
+                    return Ok(self.cx.tcx().valtree_to_const_val((ct.ty(), val)));
+                }
+                err => span_bug!(
+                    constant.span,
+                    "encountered bad ConstKind after monomorphizing: {:?}",
+                    err
+                ),
+            },
+            mir::ConstantKind::Unevaluated(uv, _) => uv,
             mir::ConstantKind::Val(val, _) => return Ok(val),
         };
-        match ct.kind() {
-            ty::ConstKind::Unevaluated(ct) => self
-                .cx
-                .tcx()
-                .const_eval_resolve(ty::ParamEnv::reveal_all(), ct, None)
-                .map_err(|err| {
-                    self.cx.tcx().sess.span_err(constant.span, "erroneous constant encountered");
-                    err
-                }),
-            ty::ConstKind::Value(val) => Ok(self.cx.tcx().valtree_to_const_val((ct.ty(), val))),
-            err => span_bug!(
-                constant.span,
-                "encountered bad ConstKind after monomorphizing: {:?}",
-                err
-            ),
-        }
+
+        self.cx.tcx().const_eval_resolve(ty::ParamEnv::reveal_all(), uv, None).map_err(|err| {
+            self.cx.tcx().sess.span_err(constant.span, "erroneous constant encountered");
+            err
+        })
     }
 
     /// process constant containing SIMD shuffle indices
diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
index 8c3186efc63..157c1c82311 100644
--- a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
@@ -3,7 +3,7 @@ use rustc_index::vec::IndexVec;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir;
 use rustc_middle::ty;
-use rustc_middle::ty::layout::LayoutOf;
+use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
 use rustc_session::config::DebugInfo;
 use rustc_span::symbol::{kw, Symbol};
 use rustc_span::{BytePos, Span};
@@ -93,15 +93,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     }
 
     /// In order to have a good line stepping behavior in debugger, we overwrite debug
-    /// locations of macro expansions with that of the outermost expansion site
-    /// (unless the crate is being compiled with `-Z debug-macros`).
+    /// locations of macro expansions with that of the outermost expansion site (when the macro is
+    /// annotated with `#[collapse_debuginfo]` or when `-Zdebug-macros` is provided).
     fn adjust_span_for_debugging(&self, mut span: Span) -> Span {
         // Bail out if debug info emission is not enabled.
         if self.debug_context.is_none() {
             return span;
         }
 
-        if span.from_expansion() && !self.cx.sess().opts.unstable_opts.debug_macros {
+        if self.cx.tcx().should_collapse_debuginfo(span) {
             // Walk up the macro expansion chain until we reach a non-expanded span.
             // We also stop at the function body level because no line stepping can occur
             // at the level above that.
diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
index 16aad07194d..215edbe02c0 100644
--- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
@@ -77,10 +77,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout);
 
         let llval = match name {
-            sym::assume => {
-                bx.assume(args[0].immediate());
-                return;
-            }
             sym::abort => {
                 bx.abort();
                 return;
@@ -555,14 +551,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 return;
             }
 
-            sym::ptr_guaranteed_eq | sym::ptr_guaranteed_ne => {
+            sym::ptr_guaranteed_cmp => {
                 let a = args[0].immediate();
                 let b = args[1].immediate();
-                if name == sym::ptr_guaranteed_eq {
-                    bx.icmp(IntPredicate::IntEQ, a, b)
-                } else {
-                    bx.icmp(IntPredicate::IntNE, a, b)
-                }
+                bx.icmp(IntPredicate::IntEQ, a, b)
             }
 
             sym::ptr_offset_from | sym::ptr_offset_from_unsigned => {
diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs
index d6bbcd99234..2b931bfc91d 100644
--- a/compiler/rustc_codegen_ssa/src/mir/mod.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs
@@ -191,7 +191,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                 // errored or at least linted
                 ErrorHandled::Reported(_) | ErrorHandled::Linted => {}
                 ErrorHandled::TooGeneric => {
-                    span_bug!(const_.span, "codgen encountered polymorphic constant: {:?}", err)
+                    span_bug!(const_.span, "codegen encountered polymorphic constant: {:?}", err)
                 }
             }
         }
diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs
index c612634fce2..37b1e036247 100644
--- a/compiler/rustc_codegen_ssa/src/mir/operand.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs
@@ -72,10 +72,6 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
     ) -> Self {
         let layout = bx.layout_of(ty);
 
-        if layout.is_zst() {
-            return OperandRef::new_zst(bx, layout);
-        }
-
         let val = match val {
             ConstValue::Scalar(x) => {
                 let Abi::Scalar(scalar) = layout.abi else {
@@ -84,10 +80,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
                 let llval = bx.scalar_to_backend(x, scalar, bx.immediate_backend_type(layout));
                 OperandValue::Immediate(llval)
             }
-            ConstValue::ZeroSized => {
-                let llval = bx.zst_to_backend(bx.immediate_backend_type(layout));
-                OperandValue::Immediate(llval)
-            }
+            ConstValue::ZeroSized => return OperandRef::new_zst(bx, layout),
             ConstValue::Slice { data, start, end } => {
                 let Abi::ScalarPair(a_scalar, _) = layout.abi else {
                     bug!("from_const: invalid ScalarPair layout: {:#?}", layout);
diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs
index 268c4d76503..9c18df5643f 100644
--- a/compiler/rustc_codegen_ssa/src/mir/place.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/place.rs
@@ -4,7 +4,6 @@ use super::{FunctionCx, LocalRef};
 use crate::common::IntPredicate;
 use crate::glue;
 use crate::traits::*;
-use crate::MemFlags;
 
 use rustc_middle::mir;
 use rustc_middle::mir::tcx::PlaceTy;
@@ -245,7 +244,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
                 };
                 bx.intcast(tag.immediate(), cast_to, signed)
             }
-            TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start } => {
+            TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => {
                 // Rebase from niche values to discriminants, and check
                 // whether the result is in range for the niche variants.
                 let niche_llty = bx.cx().immediate_backend_type(tag.layout);
@@ -303,7 +302,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
                 bx.select(
                     is_niche,
                     niche_discr,
-                    bx.cx().const_uint(cast_to, dataful_variant.as_u32() as u64),
+                    bx.cx().const_uint(cast_to, untagged_variant.as_u32() as u64),
                 )
             }
         }
@@ -338,21 +337,11 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
             }
             Variants::Multiple {
                 tag_encoding:
-                    TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start },
+                    TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start },
                 tag_field,
                 ..
             } => {
-                if variant_index != dataful_variant {
-                    if bx.cx().sess().target.arch == "arm"
-                        || bx.cx().sess().target.arch == "aarch64"
-                    {
-                        // FIXME(#34427): as workaround for LLVM bug on ARM,
-                        // use memset of 0 before assigning niche value.
-                        let fill_byte = bx.cx().const_u8(0);
-                        let size = bx.cx().const_usize(self.layout.size.bytes());
-                        bx.memset(self.llval, fill_byte, size, self.align, MemFlags::empty());
-                    }
-
+                if variant_index != untagged_variant {
                     let niche = self.project_field(bx, tag_field);
                     let niche_llty = bx.cx().immediate_backend_type(niche.layout);
                     let niche_value = variant_index.as_u32() - niche_variants.start().as_u32();
@@ -411,6 +400,21 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
         downcast
     }
 
+    pub fn project_type<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+        &self,
+        bx: &mut Bx,
+        ty: Ty<'tcx>,
+    ) -> Self {
+        let mut downcast = *self;
+        downcast.layout = bx.cx().layout_of(ty);
+
+        // Cast to the appropriate type.
+        let variant_ty = bx.cx().backend_type(downcast.layout);
+        downcast.llval = bx.pointercast(downcast.llval, bx.cx().type_ptr_to(variant_ty));
+
+        downcast
+    }
+
     pub fn storage_live<Bx: BuilderMethods<'a, 'tcx, Value = V>>(&self, bx: &mut Bx) {
         bx.lifetime_start(self.llval, self.layout.size);
     }
@@ -453,6 +457,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 mir::ProjectionElem::Field(ref field, _) => {
                     cg_base.project_field(bx, field.index())
                 }
+                mir::ProjectionElem::OpaqueCast(ty) => cg_base.project_type(bx, ty),
                 mir::ProjectionElem::Index(index) => {
                     let index = &mir::Operand::Copy(mir::Place::from(index));
                     let index = self.codegen_operand(bx, index);
diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
index 26b9fbf4428..56852b0fcc8 100644
--- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
@@ -4,6 +4,7 @@ use super::{FunctionCx, LocalRef};
 
 use crate::base;
 use crate::common::{self, IntPredicate};
+use crate::meth::get_vtable;
 use crate::traits::*;
 use crate::MemFlags;
 
@@ -87,7 +88,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     let size = bx.const_usize(dest.layout.size.bytes());
 
                     // Use llvm.memset.p0i8.* to initialize all zero arrays
-                    if bx.cx().const_to_opt_uint(v) == Some(0) {
+                    if bx.cx().const_to_opt_u128(v, false) == Some(0) {
                         let fill = bx.cx().const_u8(0);
                         bx.memset(start, fill, size, dest.align, MemFlags::empty());
                         return bx;
@@ -271,6 +272,21 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             bug!("unexpected non-pair operand");
                         }
                     }
+                    mir::CastKind::DynStar => {
+                        let data = match operand.val {
+                            OperandValue::Ref(_, _, _) => todo!(),
+                            OperandValue::Immediate(v) => v,
+                            OperandValue::Pair(_, _) => todo!(),
+                        };
+                        let trait_ref =
+                            if let ty::Dynamic(data, _, ty::DynStar) = cast.ty.kind() {
+                                data.principal()
+                            } else {
+                                bug!("Only valid to do a DynStar cast into a DynStar type")
+                            };
+                        let vtable = get_vtable(bx.cx(), source.ty(self.mir, bx.tcx()), trait_ref);
+                        OperandValue::Pair(data, vtable)
+                    }
                     mir::CastKind::Pointer(
                         PointerCast::MutToConstPointer | PointerCast::ArrayToPointer,
                     )
diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs
index f452f29883f..1db0fb3a6f1 100644
--- a/compiler/rustc_codegen_ssa/src/mir/statement.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/statement.rs
@@ -1,4 +1,5 @@
 use rustc_middle::mir;
+use rustc_middle::mir::NonDivergingIntrinsic;
 
 use super::FunctionCx;
 use super::LocalRef;
@@ -73,11 +74,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 self.codegen_coverage(&mut bx, coverage.clone(), statement.source_info.scope);
                 bx
             }
-            mir::StatementKind::CopyNonOverlapping(box mir::CopyNonOverlapping {
-                ref src,
-                ref dst,
-                ref count,
-            }) => {
+            mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(ref op)) => {
+                let op_val = self.codegen_operand(&mut bx, op);
+                bx.assume(op_val.immediate());
+                bx
+            }
+            mir::StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(
+                mir::CopyNonOverlapping { ref count, ref src, ref dst },
+            )) => {
                 let dst_val = self.codegen_operand(&mut bx, dst);
                 let src_val = self.codegen_operand(&mut bx, src);
                 let count = self.codegen_operand(&mut bx, count).immediate();
diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs
index 9062a83b8be..0e259bcd1a4 100644
--- a/compiler/rustc_codegen_ssa/src/target_features.rs
+++ b/compiler/rustc_codegen_ssa/src/target_features.rs
@@ -210,8 +210,11 @@ const POWERPC_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
     ("vsx", Some(sym::powerpc_target_feature)),
 ];
 
-const MIPS_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] =
-    &[("fp64", Some(sym::mips_target_feature)), ("msa", Some(sym::mips_target_feature))];
+const MIPS_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
+    ("fp64", Some(sym::mips_target_feature)),
+    ("msa", Some(sym::mips_target_feature)),
+    ("virt", Some(sym::mips_target_feature)),
+];
 
 const RISCV_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
     ("m", Some(sym::riscv_target_feature)),
diff --git a/compiler/rustc_codegen_ssa/src/traits/consts.rs b/compiler/rustc_codegen_ssa/src/traits/consts.rs
index 8a91d4735ba..fdc7a30e841 100644
--- a/compiler/rustc_codegen_ssa/src/traits/consts.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/consts.rs
@@ -29,7 +29,6 @@ pub trait ConstMethods<'tcx>: BackendTypes {
     fn const_data_from_alloc(&self, alloc: ConstAllocation<'tcx>) -> Self::Value;
 
     fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: Self::Type) -> Self::Value;
-    fn zst_to_backend(&self, llty: Self::Type) -> Self::Value;
     fn from_const_alloc(
         &self,
         layout: TyAndLayout<'tcx>,