Merge from rust-lang/rust

597 files changed, 13657 insertions, 10313 deletions

diff --git a/compiler/rustc_abi/src/canon_abi.rs b/compiler/rustc_abi/src/canon_abi.rs
new file mode 100644
index 00000000000..03eeb645489
--- /dev/null
+++ b/compiler/rustc_abi/src/canon_abi.rs
@@ -0,0 +1,129 @@
+use std::fmt;
+
+#[cfg(feature = "nightly")]
+use rustc_macros::HashStable_Generic;
+
+use crate::ExternAbi;
+
+/// Calling convention to determine codegen
+///
+/// CanonAbi erases certain distinctions ExternAbi preserves, but remains target-dependent.
+/// There are still both target-specific variants and aliasing variants, though much fewer.
+/// The reason for this step is the frontend may wish to show an ExternAbi but implement that ABI
+/// using a different ABI than the string per se, or describe irrelevant differences, e.g.
+/// - extern "system"
+/// - extern "cdecl"
+/// - extern "C-unwind"
+/// In that sense, this erases mere syntactic distinctions to create a canonical *directive*,
+/// rather than picking the "actual" ABI.
+#[derive(Copy, Clone, Debug)]
+#[derive(PartialOrd, Ord, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "nightly", derive(HashStable_Generic))]
+pub enum CanonAbi {
+    // NOTE: the use of nested variants for some ABIs is for many targets they don't matter,
+    // and this pushes the complexity of their reasoning to target-specific code,
+    // allowing a `match` to easily exhaustively ignore these subcategories of variants.
+    // Otherwise it is very tempting to avoid matching exhaustively!
+    C,
+    Rust,
+    RustCold,
+
+    /// ABIs relevant to 32-bit Arm targets
+    Arm(ArmCall),
+    /// ABI relevant to GPUs: the entry point for a GPU kernel
+    GpuKernel,
+
+    /// ABIs relevant to bare-metal interrupt targets
+    // FIXME(workingjubilee): a particular reason for this nesting is we might not need these?
+    // interrupt ABIs should have the same properties:
+    // - uncallable by Rust calls, as LLVM rejects it in most cases
+    // - uses a preserve-all-registers *callee* convention
+    // - should always return `-> !` (effectively... it can't use normal `ret`)
+    // what differs between targets is
+    // - allowed arguments: x86 differs slightly, having 2-3 arguments which are handled magically
+    // - may need special prologues/epilogues for some interrupts, without affecting "call ABI"
+    Interrupt(InterruptKind),
+
+    /// ABIs relevant to Windows or x86 targets
+    X86(X86Call),
+}
+
+impl fmt::Display for CanonAbi {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // convert to the ExternAbi that *shares a string* with this CanonAbi.
+        // FIXME: ideally we'd avoid printing `CanonAbi`, and preserve `ExternAbi` everywhere
+        // that we need to generate error messages.
+        let erased_abi = match self {
+            CanonAbi::C => ExternAbi::C { unwind: false },
+            CanonAbi::Rust => ExternAbi::Rust,
+            CanonAbi::RustCold => ExternAbi::RustCold,
+            CanonAbi::Arm(arm_call) => match arm_call {
+                ArmCall::Aapcs => ExternAbi::Aapcs { unwind: false },
+                ArmCall::CCmseNonSecureCall => ExternAbi::CCmseNonSecureCall,
+                ArmCall::CCmseNonSecureEntry => ExternAbi::CCmseNonSecureEntry,
+            },
+            CanonAbi::GpuKernel => ExternAbi::GpuKernel,
+            CanonAbi::Interrupt(interrupt_kind) => match interrupt_kind {
+                InterruptKind::Avr => ExternAbi::AvrInterrupt,
+                InterruptKind::AvrNonBlocking => ExternAbi::AvrNonBlockingInterrupt,
+                InterruptKind::Msp430 => ExternAbi::Msp430Interrupt,
+                InterruptKind::RiscvMachine => ExternAbi::RiscvInterruptM,
+                InterruptKind::RiscvSupervisor => ExternAbi::RiscvInterruptS,
+                InterruptKind::X86 => ExternAbi::X86Interrupt,
+            },
+            CanonAbi::X86(x86_call) => match x86_call {
+                X86Call::Fastcall => ExternAbi::Fastcall { unwind: false },
+                X86Call::Stdcall => ExternAbi::Stdcall { unwind: false },
+                X86Call::SysV64 => ExternAbi::SysV64 { unwind: false },
+                X86Call::Thiscall => ExternAbi::Thiscall { unwind: false },
+                X86Call::Vectorcall => ExternAbi::Vectorcall { unwind: false },
+                X86Call::Win64 => ExternAbi::Win64 { unwind: false },
+            },
+        };
+        erased_abi.as_str().fmt(f)
+    }
+}
+
+/// Callee codegen for interrupts
+///
+/// This is named differently from the "Call" enums because it is different:
+/// these "ABI" differences are not relevant to callers, since there is "no caller".
+/// These only affect callee codegen. making their categorization as distinct ABIs a bit peculiar.
+#[derive(Copy, Clone, Debug)]
+#[derive(PartialOrd, Ord, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "nightly", derive(HashStable_Generic))]
+pub enum InterruptKind {
+    Avr,
+    AvrNonBlocking,
+    Msp430,
+    RiscvMachine,
+    RiscvSupervisor,
+    X86,
+}
+
+/// ABIs defined for x86-{32,64}
+///
+/// One of SysV64 or Win64 may alias the C ABI, and arguably Win64 is cross-platform now?
+#[derive(Clone, Copy, Debug)]
+#[derive(PartialOrd, Ord, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "nightly", derive(HashStable_Generic))]
+pub enum X86Call {
+    /// "fastcall" has both GNU and Windows variants
+    Fastcall,
+    /// "stdcall" has both GNU and Windows variants
+    Stdcall,
+    SysV64,
+    Thiscall,
+    Vectorcall,
+    Win64,
+}
+
+/// ABIs defined for 32-bit Arm
+#[derive(Copy, Clone, Debug)]
+#[derive(PartialOrd, Ord, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "nightly", derive(HashStable_Generic))]
+pub enum ArmCall {
+    Aapcs,
+    CCmseNonSecureCall,
+    CCmseNonSecureEntry,
+}
diff --git a/compiler/rustc_abi/src/extern_abi.rs b/compiler/rustc_abi/src/extern_abi.rs
index 55f4845d216..0bc1c8a0930 100644
--- a/compiler/rustc_abi/src/extern_abi.rs
+++ b/compiler/rustc_abi/src/extern_abi.rs
@@ -7,69 +7,98 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher, StableOrd};
 #[cfg(feature = "nightly")]
 use rustc_macros::{Decodable, Encodable};
 
+use crate::AbiFromStrErr;
+
 #[cfg(test)]
 mod tests;
 
-use ExternAbi as Abi;
-
+/// ABI we expect to see within `extern "{abi}"`
 #[derive(Clone, Copy, Debug)]
 #[cfg_attr(feature = "nightly", derive(Encodable, Decodable))]
 pub enum ExternAbi {
-    // Some of the ABIs come first because every time we add a new ABI, we have to re-bless all the
-    // hashing tests. These are used in many places, so giving them stable values reduces test
-    // churn. The specific values are meaningless.
-    Rust,
+    /* universal */
+    /// presumed C ABI for the platform
     C {
         unwind: bool,
     },
-    Cdecl {
+    /// ABI of the "system" interface, e.g. the Win32 API, always "aliasing"
+    System {
         unwind: bool,
     },
-    Stdcall {
+
+    /// that's us!
+    Rust,
+    /// the mostly-unused `unboxed_closures` ABI, effectively now an impl detail unless someone
+    /// puts in the work to make it viable again... but would we need a special ABI?
+    RustCall,
+    /// For things unlikely to be called, where reducing register pressure in
+    /// `extern "Rust"` callers is worth paying extra cost in the callee.
+    /// Stronger than just `#[cold]` because `fn` pointers might be incompatible.
+    RustCold,
+
+    /// Unstable impl detail that directly uses Rust types to describe the ABI to LLVM.
+    /// Even normally-compatible Rust types can become ABI-incompatible with this ABI!
+    Unadjusted,
+
+    /// UEFI ABI, usually an alias of C, but sometimes an arch-specific alias
+    /// and only valid on platforms that have a UEFI standard
+    EfiApi,
+
+    /* arm */
+    /// Arm Architecture Procedure Call Standard, sometimes `ExternAbi::C` is an alias for this
+    Aapcs {
         unwind: bool,
     },
-    Fastcall {
+    /// extremely constrained barely-C ABI for TrustZone
+    CCmseNonSecureCall,
+    /// extremely constrained barely-C ABI for TrustZone
+    CCmseNonSecureEntry,
+
+    /* gpu */
+    /// An entry-point function called by the GPU's host
+    // FIXME: should not be callable from Rust on GPU targets, is for host's use only
+    GpuKernel,
+    /// An entry-point function called by the GPU's host
+    // FIXME: why do we have two of these?
+    PtxKernel,
+
+    /* interrupt */
+    AvrInterrupt,
+    AvrNonBlockingInterrupt,
+    Msp430Interrupt,
+    RiscvInterruptM,
+    RiscvInterruptS,
+    X86Interrupt,
+
+    /* x86 */
+    /// `ExternAbi::C` but spelled funny because x86
+    Cdecl {
         unwind: bool,
     },
-    Vectorcall {
+    /// gnu-stdcall on "unix" and win-stdcall on "windows"
+    Stdcall {
         unwind: bool,
     },
-    Thiscall {
+    /// gnu-fastcall on "unix" and win-fastcall on "windows"
+    Fastcall {
         unwind: bool,
     },
-    Aapcs {
+    /// windows C++ ABI
+    Thiscall {
         unwind: bool,
     },
-    Win64 {
+    /// uses AVX and stuff
+    Vectorcall {
         unwind: bool,
     },
+
+    /* x86_64 */
     SysV64 {
         unwind: bool,
     },
-    PtxKernel,
-    Msp430Interrupt,
-    X86Interrupt,
-    /// An entry-point function called by the GPU's host
-    // FIXME: should not be callable from Rust on GPU targets, is for host's use only
-    GpuKernel,
-    EfiApi,
-    AvrInterrupt,
-    AvrNonBlockingInterrupt,
-    CCmseNonSecureCall,
-    CCmseNonSecureEntry,
-    System {
+    Win64 {
         unwind: bool,
     },
-    RustCall,
-    /// *Not* a stable ABI, just directly use the Rust types to describe the ABI for LLVM. Even
-    /// normally ABI-compatible Rust types can become ABI-incompatible with this ABI!
-    Unadjusted,
-    /// For things unlikely to be called, where reducing register pressure in
-    /// `extern "Rust"` callers is worth paying extra cost in the callee.
-    /// Stronger than just `#[cold]` because `fn` pointers might be incompatible.
-    RustCold,
-    RiscvInterruptM,
-    RiscvInterruptS,
 }
 
 macro_rules! abi_impls {
@@ -99,11 +128,6 @@ macro_rules! abi_impls {
     }
 }
 
-#[derive(Debug)]
-pub enum AbiFromStrErr {
-    Unknown,
-}
-
 abi_impls! {
     ExternAbi = {
             C { unwind: false } =><= "C",
@@ -227,7 +251,7 @@ pub fn all_names() -> Vec<&'static str> {
 
 impl ExternAbi {
     /// Default ABI chosen for `extern fn` declarations without an explicit ABI.
-    pub const FALLBACK: Abi = Abi::C { unwind: false };
+    pub const FALLBACK: ExternAbi = ExternAbi::C { unwind: false };
 
     pub fn name(self) -> &'static str {
         self.as_str()
diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs
index 42250aa173b..58a7fcae9f6 100644
--- a/compiler/rustc_abi/src/layout.rs
+++ b/compiler/rustc_abi/src/layout.rs
@@ -8,7 +8,7 @@ use rustc_index::bit_set::BitMatrix;
 use tracing::debug;
 
 use crate::{
-    AbiAndPrefAlign, Align, BackendRepr, FieldsShape, HasDataLayout, IndexSlice, IndexVec, Integer,
+    AbiAlign, Align, BackendRepr, FieldsShape, HasDataLayout, IndexSlice, IndexVec, Integer,
     LayoutData, Niche, NonZeroUsize, Primitive, ReprOptions, Scalar, Size, StructKind, TagEncoding,
     Variants, WrappingRange,
 };
@@ -173,13 +173,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
             // Non-power-of-two vectors have padding up to the next power-of-two.
             // If we're a packed repr, remove the padding while keeping the alignment as close
             // to a vector as possible.
-            (
-                BackendRepr::Memory { sized: true },
-                AbiAndPrefAlign {
-                    abi: Align::max_aligned_factor(size),
-                    pref: dl.llvmlike_vector_align(size).pref,
-                },
-            )
+            (BackendRepr::Memory { sized: true }, AbiAlign { abi: Align::max_aligned_factor(size) })
         } else {
             (BackendRepr::SimdVector { element: e_repr, count }, dl.llvmlike_vector_align(size))
         };
@@ -435,13 +429,13 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
         }
 
         if let Some(pack) = repr.pack {
-            align = align.min(AbiAndPrefAlign::new(pack));
+            align = align.min(AbiAlign::new(pack));
         }
         // The unadjusted ABI alignment does not include repr(align), but does include repr(pack).
         // See documentation on `LayoutS::unadjusted_abi_align`.
         let unadjusted_abi_align = align.abi;
         if let Some(repr_align) = repr.align {
-            align = align.max(AbiAndPrefAlign::new(repr_align));
+            align = align.max(AbiAlign::new(repr_align));
         }
         // `align` must not be modified after this, or `unadjusted_abi_align` could be inaccurate.
         let align = align;
@@ -758,7 +752,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
                         niche_variants,
                         niche_start,
                     },
-                    tag_field: 0,
+                    tag_field: FieldIdx::new(0),
                     variants: IndexVec::new(),
                 },
                 fields: FieldsShape::Arbitrary {
@@ -1072,7 +1066,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
             variants: Variants::Multiple {
                 tag,
                 tag_encoding: TagEncoding::Direct,
-                tag_field: 0,
+                tag_field: FieldIdx::new(0),
                 variants: IndexVec::new(),
             },
             fields: FieldsShape::Arbitrary {
@@ -1289,7 +1283,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
         if let StructKind::Prefixed(prefix_size, prefix_align) = kind {
             let prefix_align =
                 if let Some(pack) = pack { prefix_align.min(pack) } else { prefix_align };
-            align = align.max(AbiAndPrefAlign::new(prefix_align));
+            align = align.max(AbiAlign::new(prefix_align));
             offset = prefix_size.align_to(prefix_align);
         }
         for &i in &inverse_memory_index {
@@ -1308,7 +1302,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
 
             // Invariant: offset < dl.obj_size_bound() <= 1<<61
             let field_align = if let Some(pack) = pack {
-                field.align.min(AbiAndPrefAlign::new(pack))
+                field.align.min(AbiAlign::new(pack))
             } else {
                 field.align
             };
@@ -1342,7 +1336,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
         // See documentation on `LayoutS::unadjusted_abi_align`.
         let unadjusted_abi_align = align.abi;
         if let Some(repr_align) = repr.align {
-            align = align.max(AbiAndPrefAlign::new(repr_align));
+            align = align.max(AbiAlign::new(repr_align));
         }
         // `align` must not be modified after this point, or `unadjusted_abi_align` could be inaccurate.
         let align = align;
diff --git a/compiler/rustc_abi/src/layout/coroutine.rs b/compiler/rustc_abi/src/layout/coroutine.rs
index 27e704d538c..2b22276d4ae 100644
--- a/compiler/rustc_abi/src/layout/coroutine.rs
+++ b/compiler/rustc_abi/src/layout/coroutine.rs
@@ -158,7 +158,7 @@ pub(super) fn layout<
     // Build a prefix layout, including "promoting" all ineligible
     // locals as part of the prefix. We compute the layout of all of
     // these fields at once to get optimal packing.
-    let tag_index = prefix_layouts.len();
+    let tag_index = prefix_layouts.next_index();
 
     // `variant_fields` already accounts for the reserved variants, so no need to add them.
     let max_discr = (variant_fields.len() - 1) as u128;
@@ -187,7 +187,7 @@ pub(super) fn layout<
 
             // "a" (`0..b_start`) and "b" (`b_start..`) correspond to
             // "outer" and "promoted" fields respectively.
-            let b_start = FieldIdx::new(tag_index + 1);
+            let b_start = tag_index.plus(1);
             let offsets_b = IndexVec::from_raw(offsets.raw.split_off(b_start.index()));
             let offsets_a = offsets;
 
diff --git a/compiler/rustc_abi/src/layout/ty.rs b/compiler/rustc_abi/src/layout/ty.rs
index 4f43c0e6f8e..bb880a58e52 100644
--- a/compiler/rustc_abi/src/layout/ty.rs
+++ b/compiler/rustc_abi/src/layout/ty.rs
@@ -5,7 +5,7 @@ use rustc_data_structures::intern::Interned;
 use rustc_macros::HashStable_Generic;
 
 use crate::{
-    AbiAndPrefAlign, Align, BackendRepr, FieldsShape, Float, HasDataLayout, LayoutData, Niche,
+    AbiAlign, Align, BackendRepr, FieldsShape, Float, HasDataLayout, LayoutData, Niche,
     PointeeInfo, Primitive, Scalar, Size, TargetDataLayout, Variants,
 };
 
@@ -39,6 +39,13 @@ rustc_index::newtype_index! {
     pub struct FieldIdx {}
 }
 
+impl FieldIdx {
+    /// The second field, at index 1.
+    ///
+    /// For use alongside [`FieldIdx::ZERO`], particularly with scalar pairs.
+    pub const ONE: FieldIdx = FieldIdx::from_u32(1);
+}
+
 rustc_index::newtype_index! {
     /// The *source-order* index of a variant in a type.
     ///
@@ -93,7 +100,7 @@ impl<'a> Layout<'a> {
         self.0.0.largest_niche
     }
 
-    pub fn align(self) -> AbiAndPrefAlign {
+    pub fn align(self) -> AbiAlign {
         self.0.0.align
     }
 
@@ -274,7 +281,7 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
 
     /// Finds the one field that is not a 1-ZST.
     /// Returns `None` if there are multiple non-1-ZST fields or only 1-ZST-fields.
-    pub fn non_1zst_field<C>(&self, cx: &C) -> Option<(usize, Self)>
+    pub fn non_1zst_field<C>(&self, cx: &C) -> Option<(FieldIdx, Self)>
     where
         Ty: TyAbiInterface<'a, C> + Copy,
     {
@@ -288,7 +295,7 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
                 // More than one non-1-ZST field.
                 return None;
             }
-            found = Some((field_idx, field));
+            found = Some((FieldIdx::from_usize(field_idx), field));
         }
         found
     }
diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs
index 59b74d29221..4268e68b2e4 100644
--- a/compiler/rustc_abi/src/lib.rs
+++ b/compiler/rustc_abi/src/lib.rs
@@ -43,7 +43,7 @@ use std::fmt;
 #[cfg(feature = "nightly")]
 use std::iter::Step;
 use std::num::{NonZeroUsize, ParseIntError};
-use std::ops::{Add, AddAssign, Mul, RangeInclusive, Sub};
+use std::ops::{Add, AddAssign, Deref, Mul, RangeInclusive, Sub};
 use std::str::FromStr;
 
 use bitflags::bitflags;
@@ -55,13 +55,14 @@ use rustc_index::{Idx, IndexSlice, IndexVec};
 use rustc_macros::{Decodable_NoContext, Encodable_NoContext, HashStable_Generic};
 
 mod callconv;
+mod canon_abi;
+mod extern_abi;
 mod layout;
 #[cfg(test)]
 mod tests;
 
-mod extern_abi;
-
 pub use callconv::{Heterogeneous, HomogeneousAggregate, Reg, RegKind};
+pub use canon_abi::{ArmCall, CanonAbi, InterruptKind, X86Call};
 pub use extern_abi::{ExternAbi, all_names};
 #[cfg(feature = "nightly")]
 pub use layout::{FIRST_VARIANT, FieldIdx, Layout, TyAbiInterface, TyAndLayout, VariantIdx};
@@ -225,22 +226,22 @@ pub const MAX_SIMD_LANES: u64 = 1 << 0xF;
 #[derive(Debug, PartialEq, Eq)]
 pub struct TargetDataLayout {
     pub endian: Endian,
-    pub i1_align: AbiAndPrefAlign,
-    pub i8_align: AbiAndPrefAlign,
-    pub i16_align: AbiAndPrefAlign,
-    pub i32_align: AbiAndPrefAlign,
-    pub i64_align: AbiAndPrefAlign,
-    pub i128_align: AbiAndPrefAlign,
-    pub f16_align: AbiAndPrefAlign,
-    pub f32_align: AbiAndPrefAlign,
-    pub f64_align: AbiAndPrefAlign,
-    pub f128_align: AbiAndPrefAlign,
+    pub i1_align: AbiAlign,
+    pub i8_align: AbiAlign,
+    pub i16_align: AbiAlign,
+    pub i32_align: AbiAlign,
+    pub i64_align: AbiAlign,
+    pub i128_align: AbiAlign,
+    pub f16_align: AbiAlign,
+    pub f32_align: AbiAlign,
+    pub f64_align: AbiAlign,
+    pub f128_align: AbiAlign,
     pub pointer_size: Size,
-    pub pointer_align: AbiAndPrefAlign,
-    pub aggregate_align: AbiAndPrefAlign,
+    pub pointer_align: AbiAlign,
+    pub aggregate_align: AbiAlign,
 
     /// Alignments for vector types.
-    pub vector_align: Vec<(Size, AbiAndPrefAlign)>,
+    pub vector_align: Vec<(Size, AbiAlign)>,
 
     pub instruction_address_space: AddressSpace,
 
@@ -256,22 +257,22 @@ impl Default for TargetDataLayout {
         let align = |bits| Align::from_bits(bits).unwrap();
         TargetDataLayout {
             endian: Endian::Big,
-            i1_align: AbiAndPrefAlign::new(align(8)),
-            i8_align: AbiAndPrefAlign::new(align(8)),
-            i16_align: AbiAndPrefAlign::new(align(16)),
-            i32_align: AbiAndPrefAlign::new(align(32)),
-            i64_align: AbiAndPrefAlign { abi: align(32), pref: align(64) },
-            i128_align: AbiAndPrefAlign { abi: align(32), pref: align(64) },
-            f16_align: AbiAndPrefAlign::new(align(16)),
-            f32_align: AbiAndPrefAlign::new(align(32)),
-            f64_align: AbiAndPrefAlign::new(align(64)),
-            f128_align: AbiAndPrefAlign::new(align(128)),
+            i1_align: AbiAlign::new(align(8)),
+            i8_align: AbiAlign::new(align(8)),
+            i16_align: AbiAlign::new(align(16)),
+            i32_align: AbiAlign::new(align(32)),
+            i64_align: AbiAlign::new(align(32)),
+            i128_align: AbiAlign::new(align(32)),
+            f16_align: AbiAlign::new(align(16)),
+            f32_align: AbiAlign::new(align(32)),
+            f64_align: AbiAlign::new(align(64)),
+            f128_align: AbiAlign::new(align(128)),
             pointer_size: Size::from_bits(64),
-            pointer_align: AbiAndPrefAlign::new(align(64)),
-            aggregate_align: AbiAndPrefAlign { abi: align(0), pref: align(64) },
+            pointer_align: AbiAlign::new(align(64)),
+            aggregate_align: AbiAlign { abi: align(8) },
             vector_align: vec![
-                (Size::from_bits(64), AbiAndPrefAlign::new(align(64))),
-                (Size::from_bits(128), AbiAndPrefAlign::new(align(128))),
+                (Size::from_bits(64), AbiAlign::new(align(64))),
+                (Size::from_bits(128), AbiAlign::new(align(128))),
             ],
             instruction_address_space: AddressSpace::DATA,
             c_enum_min_size: Integer::I32,
@@ -329,8 +330,7 @@ impl TargetDataLayout {
                     .map_err(|err| TargetDataLayoutErrors::InvalidAlignment { cause, err })
             };
             let abi = parse_bits(s[0], "alignment", cause)?;
-            let pref = s.get(1).map_or(Ok(abi), |pref| parse_bits(pref, "alignment", cause))?;
-            Ok(AbiAndPrefAlign { abi: align_from_bits(abi)?, pref: align_from_bits(pref)? })
+            Ok(AbiAlign::new(align_from_bits(abi)?))
         };
 
         let mut dl = TargetDataLayout::default();
@@ -425,7 +425,7 @@ impl TargetDataLayout {
 
     /// psABI-mandated alignment for a vector type, if any
     #[inline]
-    fn cabi_vector_align(&self, vec_size: Size) -> Option<AbiAndPrefAlign> {
+    fn cabi_vector_align(&self, vec_size: Size) -> Option<AbiAlign> {
         self.vector_align
             .iter()
             .find(|(size, _align)| *size == vec_size)
@@ -434,8 +434,8 @@ impl TargetDataLayout {
 
     /// an alignment resembling the one LLVM would pick for a vector
     #[inline]
-    pub fn llvmlike_vector_align(&self, vec_size: Size) -> AbiAndPrefAlign {
-        self.cabi_vector_align(vec_size).unwrap_or(AbiAndPrefAlign::new(
+    pub fn llvmlike_vector_align(&self, vec_size: Size) -> AbiAlign {
+        self.cabi_vector_align(vec_size).unwrap_or(AbiAlign::new(
             Align::from_bytes(vec_size.bytes().next_power_of_two()).unwrap(),
         ))
     }
@@ -863,25 +863,32 @@ impl Align {
 /// It is of effectively no consequence for layout in structs and on the stack.
 #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
 #[cfg_attr(feature = "nightly", derive(HashStable_Generic))]
-pub struct AbiAndPrefAlign {
+pub struct AbiAlign {
     pub abi: Align,
-    pub pref: Align,
 }
 
-impl AbiAndPrefAlign {
+impl AbiAlign {
     #[inline]
-    pub fn new(align: Align) -> AbiAndPrefAlign {
-        AbiAndPrefAlign { abi: align, pref: align }
+    pub fn new(align: Align) -> AbiAlign {
+        AbiAlign { abi: align }
     }
 
     #[inline]
-    pub fn min(self, other: AbiAndPrefAlign) -> AbiAndPrefAlign {
-        AbiAndPrefAlign { abi: self.abi.min(other.abi), pref: self.pref.min(other.pref) }
+    pub fn min(self, other: AbiAlign) -> AbiAlign {
+        AbiAlign { abi: self.abi.min(other.abi) }
     }
 
     #[inline]
-    pub fn max(self, other: AbiAndPrefAlign) -> AbiAndPrefAlign {
-        AbiAndPrefAlign { abi: self.abi.max(other.abi), pref: self.pref.max(other.pref) }
+    pub fn max(self, other: AbiAlign) -> AbiAlign {
+        AbiAlign { abi: self.abi.max(other.abi) }
+    }
+}
+
+impl Deref for AbiAlign {
+    type Target = Align;
+
+    fn deref(&self) -> &Self::Target {
+        &self.abi
     }
 }
 
@@ -944,7 +951,7 @@ impl Integer {
         }
     }
 
-    pub fn align<C: HasDataLayout>(self, cx: &C) -> AbiAndPrefAlign {
+    pub fn align<C: HasDataLayout>(self, cx: &C) -> AbiAlign {
         use Integer::*;
         let dl = cx.data_layout();
 
@@ -1057,7 +1064,7 @@ impl Float {
         }
     }
 
-    pub fn align<C: HasDataLayout>(self, cx: &C) -> AbiAndPrefAlign {
+    pub fn align<C: HasDataLayout>(self, cx: &C) -> AbiAlign {
         use Float::*;
         let dl = cx.data_layout();
 
@@ -1101,7 +1108,7 @@ impl Primitive {
         }
     }
 
-    pub fn align<C: HasDataLayout>(self, cx: &C) -> AbiAndPrefAlign {
+    pub fn align<C: HasDataLayout>(self, cx: &C) -> AbiAlign {
         use Primitive::*;
         let dl = cx.data_layout();
 
@@ -1224,7 +1231,7 @@ impl Scalar {
         }
     }
 
-    pub fn align(self, cx: &impl HasDataLayout) -> AbiAndPrefAlign {
+    pub fn align(self, cx: &impl HasDataLayout) -> AbiAlign {
         self.primitive().align(cx)
     }
 
@@ -1572,7 +1579,7 @@ pub enum Variants<FieldIdx: Idx, VariantIdx: Idx> {
     Multiple {
         tag: Scalar,
         tag_encoding: TagEncoding<VariantIdx>,
-        tag_field: usize,
+        tag_field: FieldIdx,
         variants: IndexVec<VariantIdx, LayoutData<FieldIdx, VariantIdx>>,
     },
 }
@@ -1730,7 +1737,7 @@ pub struct LayoutData<FieldIdx: Idx, VariantIdx: Idx> {
     /// especially in the case of by-pointer struct returns, which allocate stack even when unused.
     pub uninhabited: bool,
 
-    pub align: AbiAndPrefAlign,
+    pub align: AbiAlign,
     pub size: Size,
 
     /// The largest alignment explicitly requested with `repr(align)` on this type or any field.
@@ -1895,3 +1902,11 @@ pub enum StructKind {
     /// A univariant, but with a prefix of an arbitrary size & alignment (e.g., enum tag).
     Prefixed(Size, Align),
 }
+
+#[derive(Clone, Debug)]
+pub enum AbiFromStrErr {
+    /// not a known ABI
+    Unknown,
+    /// no "-unwind" variant can be used here
+    NoExplicitUnwind,
+}
diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
index a16219361c0..cf40c3f7f6f 100644
--- a/compiler/rustc_ast/src/ast.rs
+++ b/compiler/rustc_ast/src/ast.rs
@@ -32,7 +32,7 @@ use rustc_data_structures::tagged_ptr::Tag;
 use rustc_macros::{Decodable, Encodable, HashStable_Generic};
 pub use rustc_span::AttrId;
 use rustc_span::source_map::{Spanned, respan};
-use rustc_span::{ErrorGuaranteed, Ident, Span, Symbol, kw, sym};
+use rustc_span::{DUMMY_SP, ErrorGuaranteed, Ident, Span, Symbol, kw, sym};
 use thin_vec::{ThinVec, thin_vec};
 
 pub use crate::format::*;
@@ -99,8 +99,15 @@ pub struct Path {
 
 impl PartialEq<Symbol> for Path {
     #[inline]
-    fn eq(&self, symbol: &Symbol) -> bool {
-        matches!(&self.segments[..], [segment] if segment.ident.name == *symbol)
+    fn eq(&self, name: &Symbol) -> bool {
+        if let [segment] = self.segments.as_ref()
+            && segment.args.is_none()
+            && segment.ident.name == *name
+        {
+            true
+        } else {
+            false
+        }
     }
 }
 
@@ -120,17 +127,6 @@ impl Path {
         Path { segments: thin_vec![PathSegment::from_ident(ident)], span: ident.span, tokens: None }
     }
 
-    pub fn is_ident(&self, name: Symbol) -> bool {
-        if let [segment] = self.segments.as_ref()
-            && segment.args.is_none()
-            && segment.ident.name == name
-        {
-            true
-        } else {
-            false
-        }
-    }
-
     pub fn is_global(&self) -> bool {
         self.segments.first().is_some_and(|segment| segment.ident.name == kw::PathRoot)
     }
@@ -1123,10 +1119,9 @@ impl Stmt {
     pub fn add_trailing_semicolon(mut self) -> Self {
         self.kind = match self.kind {
             StmtKind::Expr(expr) => StmtKind::Semi(expr),
-            StmtKind::MacCall(mac) => {
-                StmtKind::MacCall(mac.map(|MacCallStmt { mac, style: _, attrs, tokens }| {
-                    MacCallStmt { mac, style: MacStmtStyle::Semicolon, attrs, tokens }
-                }))
+            StmtKind::MacCall(mut mac) => {
+                mac.style = MacStmtStyle::Semicolon;
+                StmtKind::MacCall(mac)
             }
             kind => kind,
         };
@@ -1526,6 +1521,19 @@ impl Expr {
                 | ExprKind::Struct(_)
         )
     }
+
+    /// Creates a dummy `P<Expr>`.
+    ///
+    /// Should only be used when it will be replaced afterwards or as a return value when an error was encountered.
+    pub fn dummy() -> P<Expr> {
+        P(Expr {
+            id: DUMMY_NODE_ID,
+            kind: ExprKind::Dummy,
+            span: DUMMY_SP,
+            attrs: ThinVec::new(),
+            tokens: None,
+        })
+    }
 }
 
 #[derive(Clone, Encodable, Decodable, Debug)]
@@ -1715,7 +1723,7 @@ pub enum ExprKind {
     ///
     /// Usually not written directly in user code but
     /// indirectly via the macro `core::mem::offset_of!(...)`.
-    OffsetOf(P<Ty>, P<[Ident]>),
+    OffsetOf(P<Ty>, Vec<Ident>),
 
     /// A macro invocation; pre-expansion.
     MacCall(P<MacCall>),
@@ -2452,6 +2460,39 @@ impl TyKind {
             None
         }
     }
+
+    /// Returns `true` if this type is considered a scalar primitive (e.g.,
+    /// `i32`, `u8`, `bool`, etc).
+    ///
+    /// This check is based on **symbol equality** and does **not** remove any
+    /// path prefixes or references. If a type alias or shadowing is present
+    /// (e.g., `type i32 = CustomType;`), this method will still return `true`
+    /// for `i32`, even though it may not refer to the primitive type.
+    pub fn maybe_scalar(&self) -> bool {
+        let Some(ty_sym) = self.is_simple_path() else {
+            // unit type
+            return self.is_unit();
+        };
+        matches!(
+            ty_sym,
+            sym::i8
+                | sym::i16
+                | sym::i32
+                | sym::i64
+                | sym::i128
+                | sym::u8
+                | sym::u16
+                | sym::u32
+                | sym::u64
+                | sym::u128
+                | sym::f16
+                | sym::f32
+                | sym::f64
+                | sym::f128
+                | sym::char
+                | sym::bool
+        )
+    }
 }
 
 /// A pattern type pattern.
@@ -3417,9 +3458,9 @@ impl Item {
             ItemKind::Fn(i) => Some(&i.generics),
             ItemKind::TyAlias(i) => Some(&i.generics),
             ItemKind::TraitAlias(_, generics, _)
-            | ItemKind::Enum(_, _, generics)
-            | ItemKind::Struct(_, _, generics)
-            | ItemKind::Union(_, _, generics) => Some(&generics),
+            | ItemKind::Enum(_, generics, _)
+            | ItemKind::Struct(_, generics, _)
+            | ItemKind::Union(_, generics, _) => Some(&generics),
             ItemKind::Trait(i) => Some(&i.generics),
             ItemKind::Impl(i) => Some(&i.generics),
         }
@@ -3663,15 +3704,15 @@ pub enum ItemKind {
     /// An enum definition (`enum`).
     ///
     /// E.g., `enum Foo<A, B> { C<A>, D<B> }`.
-    Enum(Ident, EnumDef, Generics),
+    Enum(Ident, Generics, EnumDef),
     /// A struct definition (`struct`).
     ///
     /// E.g., `struct Foo<A> { x: A }`.
-    Struct(Ident, VariantData, Generics),
+    Struct(Ident, Generics, VariantData),
     /// A union definition (`union`).
     ///
     /// E.g., `union Foo<A, B> { x: A, y: B }`.
-    Union(Ident, VariantData, Generics),
+    Union(Ident, Generics, VariantData),
     /// A trait declaration (`trait`).
     ///
     /// E.g., `trait Foo { .. }`, `trait Foo<T> { .. }` or `auto trait Foo {}`.
@@ -3688,10 +3729,8 @@ pub enum ItemKind {
     ///
     /// E.g., `foo!(..)`.
     MacCall(P<MacCall>),
-
     /// A macro definition.
     MacroDef(Ident, MacroDef),
-
     /// A single delegation item (`reuse`).
     ///
     /// E.g. `reuse <Type as Trait>::name { target_expr_template }`.
@@ -3767,9 +3806,9 @@ impl ItemKind {
             Self::Fn(box Fn { generics, .. })
             | Self::TyAlias(box TyAlias { generics, .. })
             | Self::Const(box ConstItem { generics, .. })
-            | Self::Enum(_, _, generics)
-            | Self::Struct(_, _, generics)
-            | Self::Union(_, _, generics)
+            | Self::Enum(_, generics, _)
+            | Self::Struct(_, generics, _)
+            | Self::Union(_, generics, _)
             | Self::Trait(box Trait { generics, .. })
             | Self::TraitAlias(_, generics, _)
             | Self::Impl(box Impl { generics, .. }) => Some(generics),
diff --git a/compiler/rustc_ast/src/ast_traits.rs b/compiler/rustc_ast/src/ast_traits.rs
index 21de7ff7719..797ab297319 100644
--- a/compiler/rustc_ast/src/ast_traits.rs
+++ b/compiler/rustc_ast/src/ast_traits.rs
@@ -304,6 +304,7 @@ impl HasAttrs for Stmt {
 }
 
 /// A newtype around an AST node that implements the traits above if the node implements them.
+#[repr(transparent)]
 pub struct AstNodeWrapper<Wrapped, Tag> {
     pub wrapped: Wrapped,
     pub tag: PhantomData<Tag>,
@@ -313,6 +314,11 @@ impl<Wrapped, Tag> AstNodeWrapper<Wrapped, Tag> {
     pub fn new(wrapped: Wrapped, _tag: Tag) -> AstNodeWrapper<Wrapped, Tag> {
         AstNodeWrapper { wrapped, tag: Default::default() }
     }
+
+    pub fn from_mut(wrapped: &mut Wrapped, _tag: Tag) -> &mut AstNodeWrapper<Wrapped, Tag> {
+        // SAFETY: `AstNodeWrapper` is `repr(transparent)` w.r.t `Wrapped`
+        unsafe { &mut *<*mut Wrapped>::cast(wrapped) }
+    }
 }
 
 impl<Wrapped: HasNodeId, Tag> HasNodeId for AstNodeWrapper<Wrapped, Tag> {
diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs
index f165c4ddcdd..621e3042b62 100644
--- a/compiler/rustc_ast/src/attr/mod.rs
+++ b/compiler/rustc_ast/src/attr/mod.rs
@@ -63,7 +63,7 @@ impl Attribute {
 
     pub fn unwrap_normal_item(self) -> AttrItem {
         match self.kind {
-            AttrKind::Normal(normal) => normal.into_inner().item,
+            AttrKind::Normal(normal) => normal.item,
             AttrKind::DocComment(..) => panic!("unexpected doc comment"),
         }
     }
diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs
index a90349f318c..77cbdde61a4 100644
--- a/compiler/rustc_ast/src/mut_visit.rs
+++ b/compiler/rustc_ast/src/mut_visit.rs
@@ -12,7 +12,6 @@ use std::panic;
 
 use rustc_data_structures::flat_map_in_place::FlatMapInPlace;
 use rustc_data_structures::stack::ensure_sufficient_stack;
-use rustc_span::source_map::Spanned;
 use rustc_span::{Ident, Span};
 use smallvec::{Array, SmallVec, smallvec};
 use thin_vec::ThinVec;
@@ -20,7 +19,7 @@ use thin_vec::ThinVec;
 use crate::ast::*;
 use crate::ptr::P;
 use crate::tokenstream::*;
-use crate::visit::{AssocCtxt, BoundKind, FnCtxt, try_visit};
+use crate::visit::{AssocCtxt, BoundKind, FnCtxt, try_visit, visit_opt, walk_list};
 
 pub trait ExpectOne<A: Array> {
     fn expect_one(self, err: &'static str) -> A::Item;
@@ -33,18 +32,6 @@ impl<A: Array> ExpectOne<A> for SmallVec<A> {
     }
 }
 
-pub trait WalkItemKind {
-    type Ctxt;
-    fn walk(
-        &mut self,
-        span: Span,
-        id: NodeId,
-        visibility: &mut Visibility,
-        ctxt: Self::Ctxt,
-        visitor: &mut impl MutVisitor,
-    );
-}
-
 pub trait MutVisitor: Sized {
     // Methods in this trait have one of three forms:
     //
@@ -52,12 +39,6 @@ pub trait MutVisitor: Sized {
     //   fn flat_map_t(&mut self, t: T) -> SmallVec<[T; 1]>;    // rare
     //   fn filter_map_t(&mut self, t: T) -> Option<T>;         // rarest
     //
-    // Any additions to this trait should happen in form of a call to a public
-    // `noop_*` function that only calls out to the visitor again, not other
-    // `noop_*` functions. This is a necessary API workaround to the problem of
-    // not being able to call out to the super default method in an overridden
-    // default method.
-    //
     // When writing these methods, it is better to use destructuring like this:
     //
     //   fn visit_abc(&mut self, ABC { a, b, c: _ }: &mut ABC) {
@@ -95,7 +76,7 @@ pub trait MutVisitor: Sized {
         walk_use_tree(self, use_tree);
     }
 
-    fn visit_foreign_item(&mut self, ni: &mut P<ForeignItem>) {
+    fn visit_foreign_item(&mut self, ni: &mut ForeignItem) {
         walk_item(self, ni);
     }
 
@@ -103,7 +84,7 @@ pub trait MutVisitor: Sized {
         walk_flat_map_foreign_item(self, ni)
     }
 
-    fn visit_item(&mut self, i: &mut P<Item>) {
+    fn visit_item(&mut self, i: &mut Item) {
         walk_item(self, i);
     }
 
@@ -123,7 +104,7 @@ pub trait MutVisitor: Sized {
         walk_flat_map_field_def(self, fd)
     }
 
-    fn visit_assoc_item(&mut self, i: &mut P<AssocItem>, ctxt: AssocCtxt) {
+    fn visit_assoc_item(&mut self, i: &mut AssocItem, ctxt: AssocCtxt) {
         walk_assoc_item(self, i, ctxt)
     }
 
@@ -135,11 +116,11 @@ pub trait MutVisitor: Sized {
         walk_flat_map_assoc_item(self, i, ctxt)
     }
 
-    fn visit_contract(&mut self, c: &mut P<FnContract>) {
+    fn visit_contract(&mut self, c: &mut FnContract) {
         walk_contract(self, c);
     }
 
-    fn visit_fn_decl(&mut self, d: &mut P<FnDecl>) {
+    fn visit_fn_decl(&mut self, d: &mut FnDecl) {
         walk_fn_decl(self, d);
     }
 
@@ -156,7 +137,7 @@ pub trait MutVisitor: Sized {
         walk_closure_binder(self, b);
     }
 
-    fn visit_block(&mut self, b: &mut P<Block>) {
+    fn visit_block(&mut self, b: &mut Block) {
         walk_block(self, b);
     }
 
@@ -191,7 +172,7 @@ pub trait MutVisitor: Sized {
     }
 
     fn filter_map_expr(&mut self, e: P<Expr>) -> Option<P<Expr>> {
-        noop_filter_map_expr(self, e)
+        walk_filter_map_expr(self, e)
     }
 
     fn visit_generic_arg(&mut self, arg: &mut GenericArg) {
@@ -202,7 +183,7 @@ pub trait MutVisitor: Sized {
         walk_ty(self, t);
     }
 
-    fn visit_ty_pat(&mut self, t: &mut P<TyPat>) {
+    fn visit_ty_pat(&mut self, t: &mut TyPat) {
         walk_ty_pat(self, t);
     }
 
@@ -227,11 +208,7 @@ pub trait MutVisitor: Sized {
     }
 
     fn visit_ident(&mut self, i: &mut Ident) {
-        walk_ident(self, i);
-    }
-
-    fn visit_modifiers(&mut self, m: &mut TraitBoundModifiers) {
-        walk_modifiers(self, m);
+        self.visit_span(&mut i.span);
     }
 
     fn visit_path(&mut self, p: &mut Path) {
@@ -258,7 +235,7 @@ pub trait MutVisitor: Sized {
         walk_parenthesized_parameter_data(self, p);
     }
 
-    fn visit_local(&mut self, l: &mut P<Local>) {
+    fn visit_local(&mut self, l: &mut Local) {
         walk_local(self, l);
     }
 
@@ -318,10 +295,6 @@ pub trait MutVisitor: Sized {
         walk_precise_capturing_arg(self, arg);
     }
 
-    fn visit_mt(&mut self, mt: &mut MutTy) {
-        walk_mt(self, mt);
-    }
-
     fn visit_expr_field(&mut self, f: &mut ExprField) {
         walk_expr_field(self, f);
     }
@@ -390,17 +363,33 @@ pub trait MutVisitor: Sized {
 
 super::common_visitor_and_walkers!((mut) MutVisitor);
 
-/// Use a map-style function (`FnOnce(T) -> T`) to overwrite a `&mut T`. Useful
-/// when using a `flat_map_*` or `filter_map_*` method within a `visit_`
-/// method.
-//
-// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
-pub fn visit_clobber<T: DummyAstNode>(t: &mut T, f: impl FnOnce(T) -> T) {
-    let old_t = std::mem::replace(t, T::dummy());
-    *t = f(old_t);
+macro_rules! generate_flat_map_visitor_fns {
+    ($($name:ident, $Ty:ty, $flat_map_fn:ident$(, $param:ident: $ParamTy:ty)*;)+) => {
+        $(
+            fn $name<V: MutVisitor>(
+                vis: &mut V,
+                values: &mut ThinVec<$Ty>,
+                $(
+                    $param: $ParamTy,
+                )*
+            ) {
+                values.flat_map_in_place(|value| vis.$flat_map_fn(value$(,$param)*));
+            }
+        )+
+    }
+}
+
+generate_flat_map_visitor_fns! {
+    visit_items, P<Item>, flat_map_item;
+    visit_foreign_items, P<ForeignItem>, flat_map_foreign_item;
+    visit_generic_params, GenericParam, flat_map_generic_param;
+    visit_stmts, Stmt, flat_map_stmt;
+    visit_exprs, P<Expr>, filter_map_expr;
+    visit_pat_fields, PatField, flat_map_pat_field;
+    visit_variants, Variant, flat_map_variant;
+    visit_assoc_items, P<AssocItem>, flat_map_assoc_item, ctxt: AssocCtxt;
 }
 
-// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
 #[inline]
 fn visit_vec<T, F>(elems: &mut Vec<T>, mut visit_elem: F)
 where
@@ -411,7 +400,6 @@ where
     }
 }
 
-// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
 #[inline]
 fn visit_thin_vec<T, F>(elems: &mut ThinVec<T>, mut visit_elem: F)
 where
@@ -422,7 +410,6 @@ where
     }
 }
 
-// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
 #[inline]
 fn visit_opt<T, F>(opt: &mut Option<T>, mut visit_elem: F)
 where
@@ -433,30 +420,12 @@ where
     }
 }
 
-// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
 fn visit_attrs<T: MutVisitor>(vis: &mut T, attrs: &mut AttrVec) {
     for attr in attrs.iter_mut() {
         vis.visit_attribute(attr);
     }
 }
 
-// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
-#[allow(unused)]
-fn visit_exprs<T: MutVisitor>(vis: &mut T, exprs: &mut Vec<P<Expr>>) {
-    exprs.flat_map_in_place(|expr| vis.filter_map_expr(expr))
-}
-
-// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
-fn visit_thin_exprs<T: MutVisitor>(vis: &mut T, exprs: &mut ThinVec<P<Expr>>) {
-    exprs.flat_map_in_place(|expr| vis.filter_map_expr(expr))
-}
-
-// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
-fn visit_bounds<T: MutVisitor>(vis: &mut T, bounds: &mut GenericBounds, ctxt: BoundKind) {
-    visit_vec(bounds, |bound| vis.visit_param_bound(bound, ctxt));
-}
-
-// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
 fn visit_attr_args<T: MutVisitor>(vis: &mut T, args: &mut AttrArgs) {
     match args {
         AttrArgs::Empty => {}
@@ -468,7 +437,6 @@ fn visit_attr_args<T: MutVisitor>(vis: &mut T, args: &mut AttrArgs) {
     }
 }
 
-// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
 fn visit_delim_args<T: MutVisitor>(vis: &mut T, args: &mut DelimArgs) {
     let DelimArgs { dspan, delim: _, tokens: _ } = args;
     let DelimSpan { open, close } = dspan;
@@ -476,15 +444,6 @@ fn visit_delim_args<T: MutVisitor>(vis: &mut T, args: &mut DelimArgs) {
     vis.visit_span(close);
 }
 
-pub fn walk_pat_field<T: MutVisitor>(vis: &mut T, fp: &mut PatField) {
-    let PatField { attrs, id, ident, is_placeholder: _, is_shorthand: _, pat, span } = fp;
-    vis.visit_id(id);
-    visit_attrs(vis, attrs);
-    vis.visit_ident(ident);
-    vis.visit_pat(pat);
-    vis.visit_span(span);
-}
-
 pub fn walk_flat_map_pat_field<T: MutVisitor>(
     vis: &mut T,
     mut fp: PatField,
@@ -493,21 +452,13 @@ pub fn walk_flat_map_pat_field<T: MutVisitor>(
     smallvec![fp]
 }
 
-fn walk_use_tree<T: MutVisitor>(vis: &mut T, use_tree: &mut UseTree) {
-    let UseTree { prefix, kind, span } = use_tree;
-    vis.visit_path(prefix);
-    match kind {
-        UseTreeKind::Simple(rename) => visit_opt(rename, |rename| vis.visit_ident(rename)),
-        UseTreeKind::Nested { items, span } => {
-            for (tree, id) in items {
-                vis.visit_id(id);
-                vis.visit_use_tree(tree);
-            }
-            vis.visit_span(span);
-        }
-        UseTreeKind::Glob => {}
-    }
-    vis.visit_span(span);
+fn visit_nested_use_tree<V: MutVisitor>(
+    vis: &mut V,
+    nested_tree: &mut UseTree,
+    nested_id: &mut NodeId,
+) {
+    vis.visit_id(nested_id);
+    vis.visit_use_tree(nested_tree);
 }
 
 pub fn walk_arm<T: MutVisitor>(vis: &mut T, arm: &mut Arm) {
@@ -544,89 +495,6 @@ fn walk_assoc_item_constraint<T: MutVisitor>(
     vis.visit_span(span);
 }
 
-pub fn walk_ty<T: MutVisitor>(vis: &mut T, ty: &mut P<Ty>) {
-    let Ty { id, kind, span, tokens: _ } = ty.deref_mut();
-    vis.visit_id(id);
-    match kind {
-        TyKind::Err(_guar) => {}
-        TyKind::Infer | TyKind::ImplicitSelf | TyKind::Dummy | TyKind::Never | TyKind::CVarArgs => {
-        }
-        TyKind::Slice(ty) => vis.visit_ty(ty),
-        TyKind::Ptr(mt) => vis.visit_mt(mt),
-        TyKind::Ref(lt, mt) | TyKind::PinnedRef(lt, mt) => {
-            visit_opt(lt, |lt| vis.visit_lifetime(lt));
-            vis.visit_mt(mt);
-        }
-        TyKind::BareFn(bft) => {
-            let BareFnTy { safety, ext: _, generic_params, decl, decl_span } = bft.deref_mut();
-            visit_safety(vis, safety);
-            generic_params.flat_map_in_place(|param| vis.flat_map_generic_param(param));
-            vis.visit_fn_decl(decl);
-            vis.visit_span(decl_span);
-        }
-        TyKind::UnsafeBinder(binder) => {
-            let UnsafeBinderTy { generic_params, inner_ty } = binder.deref_mut();
-            generic_params.flat_map_in_place(|param| vis.flat_map_generic_param(param));
-            vis.visit_ty(inner_ty);
-        }
-        TyKind::Tup(tys) => visit_thin_vec(tys, |ty| vis.visit_ty(ty)),
-        TyKind::Paren(ty) => vis.visit_ty(ty),
-        TyKind::Pat(ty, pat) => {
-            vis.visit_ty(ty);
-            vis.visit_ty_pat(pat);
-        }
-        TyKind::Path(qself, path) => {
-            vis.visit_qself(qself);
-            vis.visit_path(path);
-        }
-        TyKind::Array(ty, length) => {
-            vis.visit_ty(ty);
-            vis.visit_anon_const(length);
-        }
-        TyKind::Typeof(expr) => vis.visit_anon_const(expr),
-        TyKind::TraitObject(bounds, _syntax) => {
-            visit_vec(bounds, |bound| vis.visit_param_bound(bound, BoundKind::TraitObject))
-        }
-        TyKind::ImplTrait(id, bounds) => {
-            vis.visit_id(id);
-            visit_vec(bounds, |bound| vis.visit_param_bound(bound, BoundKind::Impl));
-        }
-        TyKind::MacCall(mac) => vis.visit_mac_call(mac),
-    }
-    vis.visit_span(span);
-}
-
-pub fn walk_ty_pat<T: MutVisitor>(vis: &mut T, ty: &mut P<TyPat>) {
-    let TyPat { id, kind, span, tokens: _ } = ty.deref_mut();
-    vis.visit_id(id);
-    match kind {
-        TyPatKind::Range(start, end, _include_end) => {
-            visit_opt(start, |c| vis.visit_anon_const(c));
-            visit_opt(end, |c| vis.visit_anon_const(c));
-        }
-        TyPatKind::Or(variants) => visit_thin_vec(variants, |p| vis.visit_ty_pat(p)),
-        TyPatKind::Err(_) => {}
-    }
-    vis.visit_span(span);
-}
-
-fn walk_foreign_mod<T: MutVisitor>(vis: &mut T, foreign_mod: &mut ForeignMod) {
-    let ForeignMod { extern_span: _, safety, abi: _, items } = foreign_mod;
-    visit_safety(vis, safety);
-    items.flat_map_in_place(|item| vis.flat_map_foreign_item(item));
-}
-
-pub fn walk_variant<T: MutVisitor>(visitor: &mut T, variant: &mut Variant) {
-    let Variant { ident, vis, attrs, id, data, disr_expr, span, is_placeholder: _ } = variant;
-    visitor.visit_id(id);
-    visit_attrs(visitor, attrs);
-    visitor.visit_vis(vis);
-    visitor.visit_ident(ident);
-    visitor.visit_variant_data(data);
-    visit_opt(disr_expr, |disr_expr| visitor.visit_anon_const(disr_expr));
-    visitor.visit_span(span);
-}
-
 pub fn walk_flat_map_variant<T: MutVisitor>(
     vis: &mut T,
     mut variant: Variant,
@@ -635,32 +503,6 @@ pub fn walk_flat_map_variant<T: MutVisitor>(
     smallvec![variant]
 }
 
-fn walk_ident<T: MutVisitor>(vis: &mut T, Ident { name: _, span }: &mut Ident) {
-    vis.visit_span(span);
-}
-
-fn walk_path_segment<T: MutVisitor>(vis: &mut T, segment: &mut PathSegment) {
-    let PathSegment { ident, id, args } = segment;
-    vis.visit_id(id);
-    vis.visit_ident(ident);
-    visit_opt(args, |args| vis.visit_generic_args(args));
-}
-
-fn walk_path<T: MutVisitor>(vis: &mut T, Path { segments, span, tokens: _ }: &mut Path) {
-    for segment in segments {
-        vis.visit_path_segment(segment);
-    }
-    vis.visit_span(span);
-}
-
-fn walk_qself<T: MutVisitor>(vis: &mut T, qself: &mut Option<P<QSelf>>) {
-    visit_opt(qself, |qself| {
-        let QSelf { ty, path_span, position: _ } = &mut **qself;
-        vis.visit_ty(ty);
-        vis.visit_span(path_span);
-    })
-}
-
 fn walk_generic_args<T: MutVisitor>(vis: &mut T, generic_args: &mut GenericArgs) {
     match generic_args {
         GenericArgs::AngleBracketed(data) => vis.visit_angle_bracketed_parameter_data(data),
@@ -694,27 +536,6 @@ fn walk_parenthesized_parameter_data<T: MutVisitor>(vis: &mut T, args: &mut Pare
     vis.visit_span(inputs_span);
 }
 
-fn walk_local<T: MutVisitor>(vis: &mut T, local: &mut P<Local>) {
-    let Local { id, super_, pat, ty, kind, span, colon_sp, attrs, tokens: _ } = local.deref_mut();
-    visit_opt(super_, |sp| vis.visit_span(sp));
-    vis.visit_id(id);
-    visit_attrs(vis, attrs);
-    vis.visit_pat(pat);
-    visit_opt(ty, |ty| vis.visit_ty(ty));
-    match kind {
-        LocalKind::Decl => {}
-        LocalKind::Init(init) => {
-            vis.visit_expr(init);
-        }
-        LocalKind::InitElse(init, els) => {
-            vis.visit_expr(init);
-            vis.visit_block(els);
-        }
-    }
-    visit_opt(colon_sp, |sp| vis.visit_span(sp));
-    vis.visit_span(span);
-}
-
 fn walk_attribute<T: MutVisitor>(vis: &mut T, attr: &mut Attribute) {
     let Attribute { kind, id: _, style: _, span } = attr;
     match kind {
@@ -771,22 +592,6 @@ pub fn walk_flat_map_param<T: MutVisitor>(vis: &mut T, mut param: Param) -> Smal
     smallvec![param]
 }
 
-// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
-fn visit_defaultness<T: MutVisitor>(vis: &mut T, defaultness: &mut Defaultness) {
-    match defaultness {
-        Defaultness::Default(span) => vis.visit_span(span),
-        Defaultness::Final => {}
-    }
-}
-
-// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
-fn visit_polarity<T: MutVisitor>(vis: &mut T, polarity: &mut ImplPolarity) {
-    match polarity {
-        ImplPolarity::Positive => {}
-        ImplPolarity::Negative(span) => vis.visit_span(span),
-    }
-}
-
 fn walk_closure_binder<T: MutVisitor>(vis: &mut T, binder: &mut ClosureBinder) {
     match binder {
         ClosureBinder::NotPresent => {}
@@ -796,18 +601,6 @@ fn walk_closure_binder<T: MutVisitor>(vis: &mut T, binder: &mut ClosureBinder) {
     }
 }
 
-fn walk_coroutine_kind<T: MutVisitor>(vis: &mut T, coroutine_kind: &mut CoroutineKind) {
-    match coroutine_kind {
-        CoroutineKind::Async { span, closure_id, return_impl_trait_id }
-        | CoroutineKind::Gen { span, closure_id, return_impl_trait_id }
-        | CoroutineKind::AsyncGen { span, closure_id, return_impl_trait_id } => {
-            vis.visit_id(closure_id);
-            vis.visit_id(return_impl_trait_id);
-            vis.visit_span(span);
-        }
-    }
-}
-
 fn walk_fn<T: MutVisitor>(vis: &mut T, kind: FnKind<'_>) {
     match kind {
         FnKind::Fn(
@@ -848,8 +641,8 @@ fn walk_fn<T: MutVisitor>(vis: &mut T, kind: FnKind<'_>) {
     }
 }
 
-fn walk_contract<T: MutVisitor>(vis: &mut T, contract: &mut P<FnContract>) {
-    let FnContract { requires, ensures } = contract.deref_mut();
+fn walk_contract<T: MutVisitor>(vis: &mut T, contract: &mut FnContract) {
+    let FnContract { requires, ensures } = contract;
     if let Some(pred) = requires {
         vis.visit_expr(pred);
     }
@@ -858,8 +651,8 @@ fn walk_contract<T: MutVisitor>(vis: &mut T, contract: &mut P<FnContract>) {
     }
 }
 
-fn walk_fn_decl<T: MutVisitor>(vis: &mut T, decl: &mut P<FnDecl>) {
-    let FnDecl { inputs, output } = decl.deref_mut();
+fn walk_fn_decl<T: MutVisitor>(vis: &mut T, decl: &mut FnDecl) {
+    let FnDecl { inputs, output } = decl;
     inputs.flat_map_in_place(|param| vis.flat_map_param(param));
     vis.visit_fn_ret_ty(output);
 }
@@ -992,35 +785,6 @@ fn walk_variant_data<T: MutVisitor>(vis: &mut T, vdata: &mut VariantData) {
     }
 }
 
-fn walk_trait_ref<T: MutVisitor>(vis: &mut T, TraitRef { path, ref_id }: &mut TraitRef) {
-    vis.visit_id(ref_id);
-    vis.visit_path(path);
-}
-
-fn walk_poly_trait_ref<T: MutVisitor>(vis: &mut T, p: &mut PolyTraitRef) {
-    let PolyTraitRef { bound_generic_params, modifiers, trait_ref, span } = p;
-    vis.visit_modifiers(modifiers);
-    bound_generic_params.flat_map_in_place(|param| vis.flat_map_generic_param(param));
-    vis.visit_trait_ref(trait_ref);
-    vis.visit_span(span);
-}
-
-fn walk_modifiers<V: MutVisitor>(vis: &mut V, m: &mut TraitBoundModifiers) {
-    let TraitBoundModifiers { constness, asyncness, polarity } = m;
-    match constness {
-        BoundConstness::Never => {}
-        BoundConstness::Always(span) | BoundConstness::Maybe(span) => vis.visit_span(span),
-    }
-    match asyncness {
-        BoundAsyncness::Normal => {}
-        BoundAsyncness::Async(span) => vis.visit_span(span),
-    }
-    match polarity {
-        BoundPolarity::Positive => {}
-        BoundPolarity::Negative(span) | BoundPolarity::Maybe(span) => vis.visit_span(span),
-    }
-}
-
 pub fn walk_field_def<T: MutVisitor>(visitor: &mut T, fd: &mut FieldDef) {
     let FieldDef { span, ident, vis, id, ty, attrs, is_placeholder: _, safety, default } = fd;
     visitor.visit_id(id);
@@ -1041,15 +805,6 @@ pub fn walk_flat_map_field_def<T: MutVisitor>(
     smallvec![fd]
 }
 
-pub fn walk_expr_field<T: MutVisitor>(vis: &mut T, f: &mut ExprField) {
-    let ExprField { ident, expr, span, is_shorthand: _, attrs, id, is_placeholder: _ } = f;
-    vis.visit_id(id);
-    visit_attrs(vis, attrs);
-    vis.visit_ident(ident);
-    vis.visit_expr(expr);
-    vis.visit_span(span);
-}
-
 pub fn walk_flat_map_expr_field<T: MutVisitor>(
     vis: &mut T,
     mut f: ExprField,
@@ -1058,17 +813,6 @@ pub fn walk_flat_map_expr_field<T: MutVisitor>(
     smallvec![f]
 }
 
-fn walk_mt<T: MutVisitor>(vis: &mut T, MutTy { ty, mutbl: _ }: &mut MutTy) {
-    vis.visit_ty(ty);
-}
-
-pub fn walk_block<T: MutVisitor>(vis: &mut T, block: &mut P<Block>) {
-    let Block { id, stmts, rules: _, span, tokens: _ } = block.deref_mut();
-    vis.visit_id(id);
-    stmts.flat_map_in_place(|stmt| vis.flat_map_stmt(stmt));
-    vis.visit_span(span);
-}
-
 pub fn walk_item_kind<K: WalkItemKind>(
     kind: &mut K,
     span: Span,
@@ -1080,273 +824,6 @@ pub fn walk_item_kind<K: WalkItemKind>(
     kind.walk(span, id, visibility, ctxt, vis)
 }
 
-impl WalkItemKind for ItemKind {
-    type Ctxt = ();
-    fn walk(
-        &mut self,
-        span: Span,
-        id: NodeId,
-        visibility: &mut Visibility,
-        _ctxt: Self::Ctxt,
-        vis: &mut impl MutVisitor,
-    ) {
-        match self {
-            ItemKind::ExternCrate(_orig_name, ident) => vis.visit_ident(ident),
-            ItemKind::Use(use_tree) => vis.visit_use_tree(use_tree),
-            ItemKind::Static(box StaticItem {
-                ident,
-                ty,
-                safety: _,
-                mutability: _,
-                expr,
-                define_opaque,
-            }) => {
-                vis.visit_ident(ident);
-                vis.visit_ty(ty);
-                visit_opt(expr, |expr| vis.visit_expr(expr));
-                walk_define_opaques(vis, define_opaque);
-            }
-            ItemKind::Const(item) => {
-                walk_const_item(vis, item);
-            }
-            ItemKind::Fn(func) => {
-                vis.visit_fn(FnKind::Fn(FnCtxt::Free, visibility, &mut *func), span, id);
-            }
-            ItemKind::Mod(safety, ident, mod_kind) => {
-                visit_safety(vis, safety);
-                vis.visit_ident(ident);
-                match mod_kind {
-                    ModKind::Loaded(
-                        items,
-                        _inline,
-                        ModSpans { inner_span, inject_use_span },
-                        _,
-                    ) => {
-                        items.flat_map_in_place(|item| vis.flat_map_item(item));
-                        vis.visit_span(inner_span);
-                        vis.visit_span(inject_use_span);
-                    }
-                    ModKind::Unloaded => {}
-                }
-            }
-            ItemKind::ForeignMod(nm) => vis.visit_foreign_mod(nm),
-            ItemKind::GlobalAsm(asm) => vis.visit_inline_asm(asm),
-            ItemKind::TyAlias(box TyAlias {
-                defaultness,
-                ident,
-                generics,
-                where_clauses,
-                bounds,
-                ty,
-            }) => {
-                visit_defaultness(vis, defaultness);
-                vis.visit_ident(ident);
-                vis.visit_generics(generics);
-                visit_bounds(vis, bounds, BoundKind::Bound);
-                visit_opt(ty, |ty| vis.visit_ty(ty));
-                walk_ty_alias_where_clauses(vis, where_clauses);
-            }
-            ItemKind::Enum(ident, EnumDef { variants }, generics) => {
-                vis.visit_ident(ident);
-                vis.visit_generics(generics);
-                variants.flat_map_in_place(|variant| vis.flat_map_variant(variant));
-            }
-            ItemKind::Struct(ident, variant_data, generics)
-            | ItemKind::Union(ident, variant_data, generics) => {
-                vis.visit_ident(ident);
-                vis.visit_generics(generics);
-                vis.visit_variant_data(variant_data);
-            }
-            ItemKind::Impl(box Impl {
-                defaultness,
-                safety,
-                generics,
-                constness,
-                polarity,
-                of_trait,
-                self_ty,
-                items,
-            }) => {
-                visit_defaultness(vis, defaultness);
-                visit_safety(vis, safety);
-                vis.visit_generics(generics);
-                visit_constness(vis, constness);
-                visit_polarity(vis, polarity);
-                visit_opt(of_trait, |trait_ref| vis.visit_trait_ref(trait_ref));
-                vis.visit_ty(self_ty);
-                items.flat_map_in_place(|item| {
-                    vis.flat_map_assoc_item(item, AssocCtxt::Impl { of_trait: of_trait.is_some() })
-                });
-            }
-            ItemKind::Trait(box Trait { safety, is_auto: _, ident, generics, bounds, items }) => {
-                visit_safety(vis, safety);
-                vis.visit_ident(ident);
-                vis.visit_generics(generics);
-                visit_bounds(vis, bounds, BoundKind::Bound);
-                items.flat_map_in_place(|item| vis.flat_map_assoc_item(item, AssocCtxt::Trait));
-            }
-            ItemKind::TraitAlias(ident, generics, bounds) => {
-                vis.visit_ident(ident);
-                vis.visit_generics(generics);
-                visit_bounds(vis, bounds, BoundKind::Bound);
-            }
-            ItemKind::MacCall(m) => vis.visit_mac_call(m),
-            ItemKind::MacroDef(ident, def) => {
-                vis.visit_ident(ident);
-                vis.visit_macro_def(def)
-            }
-            ItemKind::Delegation(box Delegation {
-                id,
-                qself,
-                path,
-                ident,
-                rename,
-                body,
-                from_glob: _,
-            }) => {
-                vis.visit_id(id);
-                vis.visit_qself(qself);
-                vis.visit_path(path);
-                vis.visit_ident(ident);
-                if let Some(rename) = rename {
-                    vis.visit_ident(rename);
-                }
-                if let Some(body) = body {
-                    vis.visit_block(body);
-                }
-            }
-            ItemKind::DelegationMac(box DelegationMac { qself, prefix, suffixes, body }) => {
-                vis.visit_qself(qself);
-                vis.visit_path(prefix);
-                if let Some(suffixes) = suffixes {
-                    for (ident, rename) in suffixes {
-                        vis.visit_ident(ident);
-                        if let Some(rename) = rename {
-                            vis.visit_ident(rename);
-                        }
-                    }
-                }
-                if let Some(body) = body {
-                    vis.visit_block(body);
-                }
-            }
-        }
-    }
-}
-
-impl WalkItemKind for AssocItemKind {
-    type Ctxt = AssocCtxt;
-    fn walk(
-        &mut self,
-        span: Span,
-        id: NodeId,
-        visibility: &mut Visibility,
-        ctxt: Self::Ctxt,
-        visitor: &mut impl MutVisitor,
-    ) {
-        match self {
-            AssocItemKind::Const(item) => {
-                walk_const_item(visitor, item);
-            }
-            AssocItemKind::Fn(func) => {
-                visitor.visit_fn(FnKind::Fn(FnCtxt::Assoc(ctxt), visibility, &mut *func), span, id);
-            }
-            AssocItemKind::Type(box TyAlias {
-                defaultness,
-                ident,
-                generics,
-                where_clauses,
-                bounds,
-                ty,
-            }) => {
-                visit_defaultness(visitor, defaultness);
-                visitor.visit_ident(ident);
-                visitor.visit_generics(generics);
-                visit_bounds(visitor, bounds, BoundKind::Bound);
-                visit_opt(ty, |ty| visitor.visit_ty(ty));
-                walk_ty_alias_where_clauses(visitor, where_clauses);
-            }
-            AssocItemKind::MacCall(mac) => visitor.visit_mac_call(mac),
-            AssocItemKind::Delegation(box Delegation {
-                id,
-                qself,
-                path,
-                ident,
-                rename,
-                body,
-                from_glob: _,
-            }) => {
-                visitor.visit_id(id);
-                visitor.visit_qself(qself);
-                visitor.visit_path(path);
-                visitor.visit_ident(ident);
-                if let Some(rename) = rename {
-                    visitor.visit_ident(rename);
-                }
-                if let Some(body) = body {
-                    visitor.visit_block(body);
-                }
-            }
-            AssocItemKind::DelegationMac(box DelegationMac { qself, prefix, suffixes, body }) => {
-                visitor.visit_qself(qself);
-                visitor.visit_path(prefix);
-                if let Some(suffixes) = suffixes {
-                    for (ident, rename) in suffixes {
-                        visitor.visit_ident(ident);
-                        if let Some(rename) = rename {
-                            visitor.visit_ident(rename);
-                        }
-                    }
-                }
-                if let Some(body) = body {
-                    visitor.visit_block(body);
-                }
-            }
-        }
-    }
-}
-
-fn walk_const_item<T: MutVisitor>(vis: &mut T, item: &mut ConstItem) {
-    let ConstItem { defaultness, ident, generics, ty, expr, define_opaque } = item;
-    visit_defaultness(vis, defaultness);
-    vis.visit_ident(ident);
-    vis.visit_generics(generics);
-    vis.visit_ty(ty);
-    visit_opt(expr, |expr| vis.visit_expr(expr));
-    walk_define_opaques(vis, define_opaque);
-}
-
-pub fn walk_crate<T: MutVisitor>(vis: &mut T, krate: &mut Crate) {
-    let Crate { attrs, items, spans, id, is_placeholder: _ } = krate;
-    vis.visit_id(id);
-    visit_attrs(vis, attrs);
-    items.flat_map_in_place(|item| vis.flat_map_item(item));
-    let ModSpans { inner_span, inject_use_span } = spans;
-    vis.visit_span(inner_span);
-    vis.visit_span(inject_use_span);
-}
-
-pub fn walk_item(visitor: &mut impl MutVisitor, item: &mut P<Item<impl WalkItemKind<Ctxt = ()>>>) {
-    walk_item_ctxt(visitor, item, ())
-}
-
-pub fn walk_assoc_item(visitor: &mut impl MutVisitor, item: &mut P<AssocItem>, ctxt: AssocCtxt) {
-    walk_item_ctxt(visitor, item, ctxt)
-}
-
-fn walk_item_ctxt<K: WalkItemKind>(
-    visitor: &mut impl MutVisitor,
-    item: &mut P<Item<K>>,
-    ctxt: K::Ctxt,
-) {
-    let Item { attrs, id, kind, vis, span, tokens: _ } = item.deref_mut();
-    visitor.visit_id(id);
-    visit_attrs(visitor, attrs);
-    visitor.visit_vis(vis);
-    kind.walk(*span, *id, vis, ctxt, visitor);
-    visitor.visit_span(span);
-}
-
 pub fn walk_flat_map_item(vis: &mut impl MutVisitor, mut item: P<Item>) -> SmallVec<[P<Item>; 1]> {
     vis.visit_item(&mut item);
     smallvec![item]
@@ -1369,104 +846,6 @@ pub fn walk_flat_map_assoc_item(
     smallvec![item]
 }
 
-impl WalkItemKind for ForeignItemKind {
-    type Ctxt = ();
-    fn walk(
-        &mut self,
-        span: Span,
-        id: NodeId,
-        visibility: &mut Visibility,
-        _ctxt: Self::Ctxt,
-        visitor: &mut impl MutVisitor,
-    ) {
-        match self {
-            ForeignItemKind::Static(box StaticItem {
-                ident,
-                ty,
-                mutability: _,
-                expr,
-                safety: _,
-                define_opaque,
-            }) => {
-                visitor.visit_ident(ident);
-                visitor.visit_ty(ty);
-                visit_opt(expr, |expr| visitor.visit_expr(expr));
-                walk_define_opaques(visitor, define_opaque);
-            }
-            ForeignItemKind::Fn(func) => {
-                visitor.visit_fn(FnKind::Fn(FnCtxt::Foreign, visibility, &mut *func), span, id);
-            }
-            ForeignItemKind::TyAlias(box TyAlias {
-                defaultness,
-                ident,
-                generics,
-                where_clauses,
-                bounds,
-                ty,
-            }) => {
-                visit_defaultness(visitor, defaultness);
-                visitor.visit_ident(ident);
-                visitor.visit_generics(generics);
-                visit_bounds(visitor, bounds, BoundKind::Bound);
-                visit_opt(ty, |ty| visitor.visit_ty(ty));
-                walk_ty_alias_where_clauses(visitor, where_clauses);
-            }
-            ForeignItemKind::MacCall(mac) => visitor.visit_mac_call(mac),
-        }
-    }
-}
-
-pub fn walk_pat<T: MutVisitor>(vis: &mut T, pat: &mut P<Pat>) {
-    let Pat { id, kind, span, tokens: _ } = pat.deref_mut();
-    vis.visit_id(id);
-    match kind {
-        PatKind::Err(_guar) => {}
-        PatKind::Missing | PatKind::Wild | PatKind::Rest | PatKind::Never => {}
-        PatKind::Ident(_binding_mode, ident, sub) => {
-            vis.visit_ident(ident);
-            visit_opt(sub, |sub| vis.visit_pat(sub));
-        }
-        PatKind::Expr(e) => vis.visit_expr(e),
-        PatKind::TupleStruct(qself, path, elems) => {
-            vis.visit_qself(qself);
-            vis.visit_path(path);
-            visit_thin_vec(elems, |elem| vis.visit_pat(elem));
-        }
-        PatKind::Path(qself, path) => {
-            vis.visit_qself(qself);
-            vis.visit_path(path);
-        }
-        PatKind::Struct(qself, path, fields, _etc) => {
-            vis.visit_qself(qself);
-            vis.visit_path(path);
-            fields.flat_map_in_place(|field| vis.flat_map_pat_field(field));
-        }
-        PatKind::Box(inner) => vis.visit_pat(inner),
-        PatKind::Deref(inner) => vis.visit_pat(inner),
-        PatKind::Ref(inner, _mutbl) => vis.visit_pat(inner),
-        PatKind::Range(e1, e2, Spanned { span: _, node: _ }) => {
-            visit_opt(e1, |e| vis.visit_expr(e));
-            visit_opt(e2, |e| vis.visit_expr(e));
-            vis.visit_span(span);
-        }
-        PatKind::Guard(p, e) => {
-            vis.visit_pat(p);
-            vis.visit_expr(e);
-        }
-        PatKind::Tuple(elems) | PatKind::Slice(elems) | PatKind::Or(elems) => {
-            visit_thin_vec(elems, |elem| vis.visit_pat(elem))
-        }
-        PatKind::Paren(inner) => vis.visit_pat(inner),
-        PatKind::MacCall(mac) => vis.visit_mac_call(mac),
-    }
-    vis.visit_span(span);
-}
-
-fn walk_anon_const<T: MutVisitor>(vis: &mut T, AnonConst { id, value }: &mut AnonConst) {
-    vis.visit_id(id);
-    vis.visit_expr(value);
-}
-
 fn walk_inline_asm<T: MutVisitor>(vis: &mut T, asm: &mut InlineAsm) {
     // FIXME: Visit spans inside all this currently ignored stuff.
     let InlineAsm {
@@ -1526,7 +905,7 @@ pub fn walk_expr<T: MutVisitor>(vis: &mut T, Expr { kind, id, span, attrs, token
     vis.visit_id(id);
     visit_attrs(vis, attrs);
     match kind {
-        ExprKind::Array(exprs) => visit_thin_exprs(vis, exprs),
+        ExprKind::Array(exprs) => visit_exprs(vis, exprs),
         ExprKind::ConstBlock(anon_const) => {
             vis.visit_anon_const(anon_const);
         }
@@ -1534,10 +913,10 @@ pub fn walk_expr<T: MutVisitor>(vis: &mut T, Expr { kind, id, span, attrs, token
             vis.visit_expr(expr);
             vis.visit_anon_const(count);
         }
-        ExprKind::Tup(exprs) => visit_thin_exprs(vis, exprs),
+        ExprKind::Tup(exprs) => visit_exprs(vis, exprs),
         ExprKind::Call(f, args) => {
             vis.visit_expr(f);
-            visit_thin_exprs(vis, args);
+            visit_exprs(vis, args);
         }
         ExprKind::MethodCall(box MethodCall {
             seg: PathSegment { ident, id, args: seg_args },
@@ -1549,7 +928,7 @@ pub fn walk_expr<T: MutVisitor>(vis: &mut T, Expr { kind, id, span, attrs, token
             vis.visit_id(id);
             vis.visit_ident(ident);
             visit_opt(seg_args, |args| vis.visit_generic_args(args));
-            visit_thin_exprs(vis, call_args);
+            visit_exprs(vis, call_args);
             vis.visit_span(span);
         }
         ExprKind::Binary(binop, lhs, rhs) => {
@@ -1716,11 +1095,9 @@ pub fn walk_expr<T: MutVisitor>(vis: &mut T, Expr { kind, id, span, attrs, token
     vis.visit_span(span);
 }
 
-pub fn noop_filter_map_expr<T: MutVisitor>(vis: &mut T, mut e: P<Expr>) -> Option<P<Expr>> {
-    Some({
-        vis.visit_expr(&mut e);
-        e
-    })
+pub fn walk_filter_map_expr<T: MutVisitor>(vis: &mut T, mut e: P<Expr>) -> Option<P<Expr>> {
+    vis.visit_expr(&mut e);
+    Some(e)
 }
 
 pub fn walk_flat_map_stmt<T: MutVisitor>(
@@ -1786,113 +1163,6 @@ fn walk_capture_by<T: MutVisitor>(vis: &mut T, capture_by: &mut CaptureBy) {
     }
 }
 
-fn walk_define_opaques<T: MutVisitor>(
-    vis: &mut T,
-    define_opaque: &mut Option<ThinVec<(NodeId, Path)>>,
-) {
-    if let Some(define_opaque) = define_opaque {
-        for (id, path) in define_opaque {
-            vis.visit_id(id);
-            vis.visit_path(path)
-        }
-    }
-}
-
-/// Some value for the AST node that is valid but possibly meaningless. Similar
-/// to `Default` but not intended for wide use. The value will never be used
-/// meaningfully, it exists just to support unwinding in `visit_clobber` in the
-/// case where its closure panics.
-pub trait DummyAstNode {
-    fn dummy() -> Self;
-}
-
-impl<T> DummyAstNode for Option<T> {
-    fn dummy() -> Self {
-        Default::default()
-    }
-}
-
-impl<T: DummyAstNode + 'static> DummyAstNode for P<T> {
-    fn dummy() -> Self {
-        P(DummyAstNode::dummy())
-    }
-}
-
-impl DummyAstNode for Item {
-    fn dummy() -> Self {
-        Item {
-            attrs: Default::default(),
-            id: DUMMY_NODE_ID,
-            span: Default::default(),
-            vis: Visibility {
-                kind: VisibilityKind::Public,
-                span: Default::default(),
-                tokens: Default::default(),
-            },
-            kind: ItemKind::ExternCrate(None, Ident::dummy()),
-            tokens: Default::default(),
-        }
-    }
-}
-
-impl DummyAstNode for Expr {
-    fn dummy() -> Self {
-        Expr {
-            id: DUMMY_NODE_ID,
-            kind: ExprKind::Dummy,
-            span: Default::default(),
-            attrs: Default::default(),
-            tokens: Default::default(),
-        }
-    }
-}
-
-impl DummyAstNode for Ty {
-    fn dummy() -> Self {
-        Ty {
-            id: DUMMY_NODE_ID,
-            kind: TyKind::Dummy,
-            span: Default::default(),
-            tokens: Default::default(),
-        }
-    }
-}
-
-impl DummyAstNode for Pat {
-    fn dummy() -> Self {
-        Pat {
-            id: DUMMY_NODE_ID,
-            kind: PatKind::Wild,
-            span: Default::default(),
-            tokens: Default::default(),
-        }
-    }
-}
-
-impl DummyAstNode for Stmt {
-    fn dummy() -> Self {
-        Stmt { id: DUMMY_NODE_ID, kind: StmtKind::Empty, span: Default::default() }
-    }
-}
-
-impl DummyAstNode for Crate {
-    fn dummy() -> Self {
-        Crate {
-            attrs: Default::default(),
-            items: Default::default(),
-            spans: Default::default(),
-            id: DUMMY_NODE_ID,
-            is_placeholder: Default::default(),
-        }
-    }
-}
-
-impl<N: DummyAstNode, T: DummyAstNode> DummyAstNode for crate::ast_traits::AstNodeWrapper<N, T> {
-    fn dummy() -> Self {
-        crate::ast_traits::AstNodeWrapper::new(N::dummy(), T::dummy())
-    }
-}
-
 #[derive(Debug)]
 pub enum FnKind<'a> {
     /// E.g., `fn foo()`, `fn foo(&self)`, or `extern "Abi" fn foo()`.
diff --git a/compiler/rustc_ast/src/ptr.rs b/compiler/rustc_ast/src/ptr.rs
index dd923305cdf..fffeab8bbca 100644
--- a/compiler/rustc_ast/src/ptr.rs
+++ b/compiler/rustc_ast/src/ptr.rs
@@ -1,209 +1,11 @@
-//! The AST pointer.
-//!
-//! Provides [`P<T>`][struct@P], an owned smart pointer.
-//!
-//! # Motivations and benefits
-//!
-//! * **Identity**: sharing AST nodes is problematic for the various analysis
-//!   passes (e.g., one may be able to bypass the borrow checker with a shared
-//!   `ExprKind::AddrOf` node taking a mutable borrow).
-//!
-//! * **Efficiency**: folding can reuse allocation space for `P<T>` and `Vec<T>`,
-//!   the latter even when the input and output types differ (as it would be the
-//!   case with arenas or a GADT AST using type parameters to toggle features).
-//!
-//! * **Maintainability**: `P<T>` provides an interface, which can remain fully
-//!   functional even if the implementation changes (using a special thread-local
-//!   heap, for example). Moreover, a switch to, e.g., `P<'a, T>` would be easy
-//!   and mostly automated.
-
-use std::fmt::{self, Debug, Display};
-use std::ops::{Deref, DerefMut};
-use std::{slice, vec};
-
-use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
-/// An owned smart pointer.
+/// A pointer type that uniquely owns a heap allocation of type T.
 ///
-/// See the [module level documentation][crate::ptr] for details.
-pub struct P<T: ?Sized> {
-    ptr: Box<T>,
-}
+/// This used to be its own type, but now it's just a typedef for `Box` and we are planning to
+/// remove it soon.
+pub type P<T> = Box<T>;
 
 /// Construct a `P<T>` from a `T` value.
 #[allow(non_snake_case)]
-pub fn P<T: 'static>(value: T) -> P<T> {
-    P { ptr: Box::new(value) }
-}
-
-impl<T: 'static> P<T> {
-    /// Move out of the pointer.
-    /// Intended for chaining transformations not covered by `map`.
-    pub fn and_then<U, F>(self, f: F) -> U
-    where
-        F: FnOnce(T) -> U,
-    {
-        f(*self.ptr)
-    }
-
-    /// Equivalent to `and_then(|x| x)`.
-    pub fn into_inner(self) -> T {
-        *self.ptr
-    }
-
-    /// Produce a new `P<T>` from `self` without reallocating.
-    pub fn map<F>(mut self, f: F) -> P<T>
-    where
-        F: FnOnce(T) -> T,
-    {
-        let x = f(*self.ptr);
-        *self.ptr = x;
-
-        self
-    }
-
-    /// Optionally produce a new `P<T>` from `self` without reallocating.
-    pub fn filter_map<F>(mut self, f: F) -> Option<P<T>>
-    where
-        F: FnOnce(T) -> Option<T>,
-    {
-        *self.ptr = f(*self.ptr)?;
-        Some(self)
-    }
-}
-
-impl<T: ?Sized> Deref for P<T> {
-    type Target = T;
-
-    fn deref(&self) -> &T {
-        &self.ptr
-    }
-}
-
-impl<T: ?Sized> DerefMut for P<T> {
-    fn deref_mut(&mut self) -> &mut T {
-        &mut self.ptr
-    }
-}
-
-impl<T: 'static + Clone> Clone for P<T> {
-    fn clone(&self) -> P<T> {
-        P((**self).clone())
-    }
-}
-
-impl<T: ?Sized + Debug> Debug for P<T> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        Debug::fmt(&self.ptr, f)
-    }
-}
-
-impl<T: Display> Display for P<T> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        Display::fmt(&**self, f)
-    }
-}
-
-impl<T> fmt::Pointer for P<T> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        fmt::Pointer::fmt(&self.ptr, f)
-    }
-}
-
-impl<D: Decoder, T: 'static + Decodable<D>> Decodable<D> for P<T> {
-    fn decode(d: &mut D) -> P<T> {
-        P(Decodable::decode(d))
-    }
-}
-
-impl<S: Encoder, T: Encodable<S>> Encodable<S> for P<T> {
-    fn encode(&self, s: &mut S) {
-        (**self).encode(s);
-    }
-}
-
-impl<T> P<[T]> {
-    // FIXME(const-hack) make this const again
-    pub fn new() -> P<[T]> {
-        P { ptr: Box::default() }
-    }
-
-    #[inline(never)]
-    pub fn from_vec(v: Vec<T>) -> P<[T]> {
-        P { ptr: v.into_boxed_slice() }
-    }
-
-    #[inline(never)]
-    pub fn into_vec(self) -> Vec<T> {
-        self.ptr.into_vec()
-    }
-}
-
-impl<T> Default for P<[T]> {
-    /// Creates an empty `P<[T]>`.
-    fn default() -> P<[T]> {
-        P::new()
-    }
-}
-
-impl<T: Clone> Clone for P<[T]> {
-    fn clone(&self) -> P<[T]> {
-        P::from_vec(self.to_vec())
-    }
-}
-
-impl<T> From<Vec<T>> for P<[T]> {
-    fn from(v: Vec<T>) -> Self {
-        P::from_vec(v)
-    }
-}
-
-impl<T> From<P<[T]>> for Vec<T> {
-    fn from(val: P<[T]>) -> Self {
-        val.into_vec()
-    }
-}
-
-impl<T> FromIterator<T> for P<[T]> {
-    fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> P<[T]> {
-        P::from_vec(iter.into_iter().collect())
-    }
-}
-
-impl<T> IntoIterator for P<[T]> {
-    type Item = T;
-    type IntoIter = vec::IntoIter<T>;
-
-    fn into_iter(self) -> Self::IntoIter {
-        self.into_vec().into_iter()
-    }
-}
-
-impl<'a, T> IntoIterator for &'a P<[T]> {
-    type Item = &'a T;
-    type IntoIter = slice::Iter<'a, T>;
-    fn into_iter(self) -> Self::IntoIter {
-        self.ptr.iter()
-    }
-}
-
-impl<S: Encoder, T: Encodable<S>> Encodable<S> for P<[T]> {
-    fn encode(&self, s: &mut S) {
-        Encodable::encode(&**self, s);
-    }
-}
-
-impl<D: Decoder, T: Decodable<D>> Decodable<D> for P<[T]> {
-    fn decode(d: &mut D) -> P<[T]> {
-        P::from_vec(Decodable::decode(d))
-    }
-}
-
-impl<CTX, T> HashStable<CTX> for P<T>
-where
-    T: ?Sized + HashStable<CTX>,
-{
-    fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
-        (**self).hash_stable(hcx, hasher);
-    }
+pub fn P<T>(value: T) -> P<T> {
+    Box::new(value)
 }
diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs
index 636c26bcde0..3c231be20dc 100644
--- a/compiler/rustc_ast/src/tokenstream.rs
+++ b/compiler/rustc_ast/src/tokenstream.rs
@@ -57,7 +57,9 @@ impl TokenTree {
         match (self, other) {
             (TokenTree::Token(token, _), TokenTree::Token(token2, _)) => token.kind == token2.kind,
             (TokenTree::Delimited(.., delim, tts), TokenTree::Delimited(.., delim2, tts2)) => {
-                delim == delim2 && tts.eq_unspanned(tts2)
+                delim == delim2
+                    && tts.len() == tts2.len()
+                    && tts.iter().zip(tts2.iter()).all(|(a, b)| a.eq_unspanned(b))
             }
             _ => false,
         }
@@ -694,18 +696,6 @@ impl TokenStream {
         TokenStreamIter::new(self)
     }
 
-    /// Compares two `TokenStream`s, checking equality without regarding span information.
-    pub fn eq_unspanned(&self, other: &TokenStream) -> bool {
-        let mut iter1 = self.iter();
-        let mut iter2 = other.iter();
-        for (tt1, tt2) in iter::zip(&mut iter1, &mut iter2) {
-            if !tt1.eq_unspanned(tt2) {
-                return false;
-            }
-        }
-        iter1.next().is_none() && iter2.next().is_none()
-    }
-
     /// Create a token stream containing a single token with alone spacing. The
     /// spacing used for the final token in a constructed stream doesn't matter
     /// because it's never used. In practice we arbitrarily use
diff --git a/compiler/rustc_ast/src/visit.rs b/compiler/rustc_ast/src/visit.rs
index e43d7ae065d..d2f22b04a67 100644
--- a/compiler/rustc_ast/src/visit.rs
+++ b/compiler/rustc_ast/src/visit.rs
@@ -112,18 +112,6 @@ pub enum LifetimeCtxt {
     GenericArg,
 }
 
-pub trait WalkItemKind {
-    type Ctxt;
-    fn walk<'a, V: Visitor<'a>>(
-        &'a self,
-        span: Span,
-        id: NodeId,
-        visibility: &'a Visibility,
-        ctxt: Self::Ctxt,
-        visitor: &mut V,
-    ) -> V::Result;
-}
-
 /// Each method of the `Visitor` trait is a hook to be potentially
 /// overridden. Each method's default implementation recursively visits
 /// the substructure of the input via the corresponding `walk` method;
@@ -133,6 +121,10 @@ pub trait WalkItemKind {
 /// explicitly, you need to override each method. (And you also need
 /// to monitor future changes to `Visitor` in case a new method with a
 /// new default implementation gets introduced.)
+///
+/// Every `walk_*` method uses deconstruction to access fields of structs and
+/// enums. This will result in a compile error if a field is added, which makes
+/// it more likely the appropriate visit call will be added for it.
 pub trait Visitor<'ast>: Sized {
     /// The result type of the `visit_*` methods. Can be either `()`,
     /// or `ControlFlow<T>`.
@@ -141,6 +133,9 @@ pub trait Visitor<'ast>: Sized {
     fn visit_ident(&mut self, _ident: &'ast Ident) -> Self::Result {
         Self::Result::output()
     }
+    fn visit_foreign_mod(&mut self, nm: &'ast ForeignMod) -> Self::Result {
+        walk_foreign_mod(self, nm)
+    }
     fn visit_foreign_item(&mut self, i: &'ast ForeignItem) -> Self::Result {
         walk_item(self, i)
     }
@@ -219,14 +214,11 @@ pub trait Visitor<'ast>: Sized {
         walk_poly_trait_ref(self, t)
     }
     fn visit_variant_data(&mut self, s: &'ast VariantData) -> Self::Result {
-        walk_struct_def(self, s)
+        walk_variant_data(self, s)
     }
     fn visit_field_def(&mut self, s: &'ast FieldDef) -> Self::Result {
         walk_field_def(self, s)
     }
-    fn visit_enum_def(&mut self, enum_definition: &'ast EnumDef) -> Self::Result {
-        walk_enum_def(self, enum_definition)
-    }
     fn visit_variant(&mut self, v: &'ast Variant) -> Self::Result {
         walk_variant(self, v)
     }
@@ -242,19 +234,21 @@ pub trait Visitor<'ast>: Sized {
     fn visit_mac_call(&mut self, mac: &'ast MacCall) -> Self::Result {
         walk_mac(self, mac)
     }
-    fn visit_mac_def(&mut self, _mac: &'ast MacroDef, _id: NodeId) -> Self::Result {
+    fn visit_id(&mut self, _id: NodeId) -> Self::Result {
         Self::Result::output()
     }
-    fn visit_path(&mut self, path: &'ast Path, _id: NodeId) -> Self::Result {
+    fn visit_macro_def(&mut self, _mac: &'ast MacroDef) -> Self::Result {
+        Self::Result::output()
+    }
+    fn visit_path(&mut self, path: &'ast Path) -> Self::Result {
         walk_path(self, path)
     }
-    fn visit_use_tree(
-        &mut self,
-        use_tree: &'ast UseTree,
-        id: NodeId,
-        _nested: bool,
-    ) -> Self::Result {
-        walk_use_tree(self, use_tree, id)
+    fn visit_use_tree(&mut self, use_tree: &'ast UseTree) -> Self::Result {
+        walk_use_tree(self, use_tree)
+    }
+    fn visit_nested_use_tree(&mut self, use_tree: &'ast UseTree, id: NodeId) -> Self::Result {
+        try_visit!(self.visit_id(id));
+        self.visit_use_tree(use_tree)
     }
     fn visit_path_segment(&mut self, path_segment: &'ast PathSegment) -> Self::Result {
         walk_path_segment(self, path_segment)
@@ -304,8 +298,8 @@ pub trait Visitor<'ast>: Sized {
     fn visit_capture_by(&mut self, _capture_by: &'ast CaptureBy) -> Self::Result {
         Self::Result::output()
     }
-    fn visit_coroutine_kind(&mut self, _coroutine_kind: &'ast CoroutineKind) -> Self::Result {
-        Self::Result::output()
+    fn visit_coroutine_kind(&mut self, coroutine_kind: &'ast CoroutineKind) -> Self::Result {
+        walk_coroutine_kind(self, coroutine_kind)
     }
     fn visit_fn_decl(&mut self, fn_decl: &'ast FnDecl) -> Self::Result {
         walk_fn_decl(self, fn_decl)
@@ -318,6 +312,18 @@ pub trait Visitor<'ast>: Sized {
 #[macro_export]
 macro_rules! common_visitor_and_walkers {
     ($(($mut: ident))? $Visitor:ident$(<$lt:lifetime>)?) => {
+        pub trait WalkItemKind {
+            type Ctxt;
+            fn walk<$($lt,)? V: $Visitor$(<$lt>)?>(
+                &$($lt)? $($mut)? self,
+                span: Span,
+                id: NodeId,
+                visibility: &$($lt)? $($mut)? Visibility,
+                ctxt: Self::Ctxt,
+                visitor: &mut V,
+            ) $(-> <V as Visitor<$lt>>::Result)?;
+        }
+
         // this is only used by the MutVisitor. We include this symmetry here to make writing other functions easier
         $(${ignore($lt)}
             #[expect(unused, rustc::pass_by_value)]
@@ -325,23 +331,20 @@ macro_rules! common_visitor_and_walkers {
         )?
         fn visit_span<$($lt,)? V: $Visitor$(<$lt>)?>(visitor: &mut V, span: &$($lt)? $($mut)? Span) $(-> <V as Visitor<$lt>>::Result)? {
             $(
-                let _ = stringify!($mut);
+                ${ignore($mut)}
                 visitor.visit_span(span);
             )?
             $(${ignore($lt)}V::Result::output())?
         }
 
-        // this is only used by the MutVisitor. We include this symmetry here to make writing other functions easier
+        /// helper since `Visitor` wants `NodeId` but `MutVisitor` wants `&mut NodeId`
         $(${ignore($lt)}
-            #[expect(unused, rustc::pass_by_value)]
-            #[inline]
+            #[expect(rustc::pass_by_value)]
         )?
+        #[inline]
         fn visit_id<$($lt,)? V: $Visitor$(<$lt>)?>(visitor: &mut V, id: &$($lt)? $($mut)? NodeId) $(-> <V as Visitor<$lt>>::Result)? {
-            $(
-                let _ = stringify!($mut);
-                visitor.visit_id(id);
-            )?
-            $(${ignore($lt)}V::Result::output())?
+            // deref `&NodeId` into `NodeId` only for `Visitor`
+            visitor.visit_id( $(${ignore($lt)} * )? id)
         }
 
         // this is only used by the MutVisitor. We include this symmetry here to make writing other functions easier
@@ -362,6 +365,53 @@ macro_rules! common_visitor_and_walkers {
             }
         }
 
+        fn visit_defaultness<$($lt,)? V: $Visitor$(<$lt>)?>(vis: &mut V, defaultness: &$($lt)? $($mut)? Defaultness) $(-> <V as Visitor<$lt>>::Result)? {
+            match defaultness {
+                Defaultness::Default(span) => visit_span(vis, span),
+                Defaultness::Final => {
+                    $(<V as Visitor<$lt>>::Result::output())?
+                }
+            }
+        }
+
+        fn visit_polarity<$($lt,)? V: $Visitor$(<$lt>)?>(
+            vis: &mut V,
+            polarity: &$($lt)? $($mut)? ImplPolarity,
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            match polarity {
+                ImplPolarity::Positive => { $(<V as Visitor<$lt>>::Result::output())? }
+                ImplPolarity::Negative(span) => visit_span(vis, span),
+            }
+        }
+
+        $(${ignore($lt)}
+            #[inline]
+        )?
+        fn visit_modifiers<$($lt,)? V: $Visitor$(<$lt>)?>(
+            vis: &mut V,
+            m: &$($lt)? $($mut)? TraitBoundModifiers
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            let TraitBoundModifiers { constness, asyncness, polarity } = m;
+            match constness {
+                BoundConstness::Never => {}
+                BoundConstness::Always(span) | BoundConstness::Maybe(span) => try_visit!(visit_span(vis, span)),
+            }
+            match asyncness {
+                BoundAsyncness::Normal => {}
+                BoundAsyncness::Async(span) => try_visit!(visit_span(vis, span)),
+            }
+            match polarity {
+                BoundPolarity::Positive => {}
+                BoundPolarity::Negative(span) | BoundPolarity::Maybe(span) => try_visit!(visit_span(vis, span)),
+            }
+            $(<V as Visitor<$lt>>::Result::output())?
+        }
+
+        fn visit_bounds<$($lt,)? V: $Visitor$(<$lt>)?>(visitor: &mut V, bounds: &$($lt)? $($mut)? GenericBounds, ctxt: BoundKind) $(-> <V as Visitor<$lt>>::Result)? {
+            walk_list!(visitor, visit_param_bound, bounds, ctxt);
+            $(<V as Visitor<$lt>>::Result::output())?
+        }
+
         pub fn walk_label<$($lt,)? V: $Visitor$(<$lt>)?>(visitor: &mut V, Label { ident }: &$($lt)? $($mut)? Label) $(-> <V as Visitor<$lt>>::Result)? {
             visitor.visit_ident(ident)
         }
@@ -369,9 +419,7 @@ macro_rules! common_visitor_and_walkers {
         pub fn walk_fn_header<$($lt,)? V: $Visitor$(<$lt>)?>(visitor: &mut V, header: &$($lt)? $($mut)? FnHeader) $(-> <V as Visitor<$lt>>::Result)? {
             let FnHeader { safety, coroutine_kind, constness, ext: _ } = header;
             try_visit!(visit_constness(visitor, constness));
-            if let Some(coroutine_kind) = coroutine_kind {
-                try_visit!(visitor.visit_coroutine_kind(coroutine_kind));
-            }
+            visit_opt!(visitor, visit_coroutine_kind, coroutine_kind);
             visit_safety(visitor, safety)
         }
 
@@ -379,345 +427,693 @@ macro_rules! common_visitor_and_walkers {
             try_visit!(visit_id(visitor, id));
             visitor.visit_ident(ident)
         }
-    };
-}
-
-common_visitor_and_walkers!(Visitor<'a>);
-
-pub fn walk_crate<'a, V: Visitor<'a>>(visitor: &mut V, krate: &'a Crate) -> V::Result {
-    let Crate { attrs, items, spans: _, id: _, is_placeholder: _ } = krate;
-    walk_list!(visitor, visit_attribute, attrs);
-    walk_list!(visitor, visit_item, items);
-    V::Result::output()
-}
-
-pub fn walk_local<'a, V: Visitor<'a>>(visitor: &mut V, local: &'a Local) -> V::Result {
-    let Local { id: _, super_: _, pat, ty, kind, span: _, colon_sp: _, attrs, tokens: _ } = local;
-    walk_list!(visitor, visit_attribute, attrs);
-    try_visit!(visitor.visit_pat(pat));
-    visit_opt!(visitor, visit_ty, ty);
-    if let Some((init, els)) = kind.init_else_opt() {
-        try_visit!(visitor.visit_expr(init));
-        visit_opt!(visitor, visit_block, els);
-    }
-    V::Result::output()
-}
-
-pub fn walk_poly_trait_ref<'a, V>(visitor: &mut V, trait_ref: &'a PolyTraitRef) -> V::Result
-where
-    V: Visitor<'a>,
-{
-    let PolyTraitRef { bound_generic_params, modifiers: _, trait_ref, span: _ } = trait_ref;
-    walk_list!(visitor, visit_generic_param, bound_generic_params);
-    visitor.visit_trait_ref(trait_ref)
-}
-
-pub fn walk_trait_ref<'a, V: Visitor<'a>>(visitor: &mut V, trait_ref: &'a TraitRef) -> V::Result {
-    let TraitRef { path, ref_id } = trait_ref;
-    visitor.visit_path(path, *ref_id)
-}
 
-impl WalkItemKind for ItemKind {
-    type Ctxt = ();
-    fn walk<'a, V: Visitor<'a>>(
-        &'a self,
-        span: Span,
-        id: NodeId,
-        vis: &'a Visibility,
-        _ctxt: Self::Ctxt,
-        visitor: &mut V,
-    ) -> V::Result {
-        match self {
-            ItemKind::ExternCrate(_rename, ident) => try_visit!(visitor.visit_ident(ident)),
-            ItemKind::Use(use_tree) => try_visit!(visitor.visit_use_tree(use_tree, id, false)),
-            ItemKind::Static(box StaticItem {
-                ident,
-                ty,
-                safety: _,
-                mutability: _,
-                expr,
-                define_opaque,
-            }) => {
-                try_visit!(visitor.visit_ident(ident));
-                try_visit!(visitor.visit_ty(ty));
-                visit_opt!(visitor, visit_expr, expr);
-                try_visit!(walk_define_opaques(visitor, define_opaque));
-            }
-            ItemKind::Const(box ConstItem {
-                defaultness: _,
-                ident,
-                generics,
-                ty,
-                expr,
-                define_opaque,
-            }) => {
-                try_visit!(visitor.visit_ident(ident));
-                try_visit!(visitor.visit_generics(generics));
-                try_visit!(visitor.visit_ty(ty));
-                visit_opt!(visitor, visit_expr, expr);
-                try_visit!(walk_define_opaques(visitor, define_opaque));
+        fn walk_item_ctxt<$($lt,)? V: $Visitor$(<$lt>)?, K: WalkItemKind>(
+            visitor: &mut V,
+            item: &$($mut)? $($lt)? Item<K>,
+            ctxt: K::Ctxt,
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            let Item { attrs, id, kind, vis, span, tokens: _ } = item;
+            try_visit!(visit_id(visitor, id));
+            walk_list!(visitor, visit_attribute, attrs);
+            try_visit!(visitor.visit_vis(vis));
+            try_visit!(kind.walk(*span, *id, vis, ctxt, visitor));
+            visit_span(visitor, span)
+        }
+
+        pub fn walk_item<$($lt,)? V: $Visitor$(<$lt>)?, K: WalkItemKind<Ctxt = ()>>(
+            visitor: &mut V,
+            item: &$($mut)? $($lt)? Item<K>,
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            walk_item_ctxt(visitor, item, ())
+        }
+
+        pub fn walk_assoc_item<$($lt,)? V: $Visitor$(<$lt>)?>(
+            visitor: &mut V,
+            item: &$($mut)? $($lt)? AssocItem,
+            ctxt: AssocCtxt,
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            walk_item_ctxt(visitor, item, ctxt)
+        }
+
+        impl WalkItemKind for ItemKind {
+            type Ctxt = ();
+            fn walk<$($lt,)? V: $Visitor$(<$lt>)?>(
+                &$($lt)? $($mut)? self,
+                span: Span,
+                id: NodeId,
+                visibility: &$($lt)? $($mut)? Visibility,
+                _ctxt: Self::Ctxt,
+                vis: &mut V,
+            ) $(-> <V as Visitor<$lt>>::Result)? {
+                match self {
+                    ItemKind::ExternCrate(_orig_name, ident) => vis.visit_ident(ident),
+                    ItemKind::Use(use_tree) => vis.visit_use_tree(use_tree),
+                    ItemKind::Static(box StaticItem {
+                        ident,
+                        ty,
+                        safety: _,
+                        mutability: _,
+                        expr,
+                        define_opaque,
+                    }) => {
+                        try_visit!(vis.visit_ident(ident));
+                        try_visit!(vis.visit_ty(ty));
+                        visit_opt!(vis, visit_expr, expr);
+                        walk_define_opaques(vis, define_opaque)
+                    }
+                    ItemKind::Const(item) => {
+                        walk_const_item(vis, item)
+                    }
+                    ItemKind::Fn(func) => {
+                        let kind = FnKind::Fn(FnCtxt::Free, visibility, &$($mut)? *func);
+                        vis.visit_fn(kind, span, id)
+                    }
+                    ItemKind::Mod(safety, ident, mod_kind) => {
+                        try_visit!(visit_safety(vis, safety));
+                        try_visit!(vis.visit_ident(ident));
+                        match mod_kind {
+                            ModKind::Loaded(
+                                items,
+                                _inline,
+                                ModSpans { inner_span, inject_use_span },
+                                _,
+                            ) => {
+                                try_visit!(visit_items(vis, items));
+                                try_visit!(visit_span(vis, inner_span));
+                                try_visit!(visit_span(vis, inject_use_span));
+                            }
+                            ModKind::Unloaded => {}
+                        }
+                        $(<V as Visitor<$lt>>::Result::output())?
+                    }
+                    ItemKind::ForeignMod(nm) => vis.visit_foreign_mod(nm),
+                    ItemKind::GlobalAsm(asm) => vis.visit_inline_asm(asm),
+                    ItemKind::TyAlias(box TyAlias {
+                        defaultness,
+                        ident,
+                        generics,
+                        $(${ignore($lt)} #[expect(unused)])?
+                        where_clauses,
+                        bounds,
+                        ty,
+                    }) => {
+                        try_visit!(visit_defaultness(vis, defaultness));
+                        try_visit!(vis.visit_ident(ident));
+                        try_visit!(vis.visit_generics(generics));
+                        try_visit!(visit_bounds(vis, bounds, BoundKind::Bound));
+                        visit_opt!(vis, visit_ty, ty);
+                        $(${ignore($mut)}
+                            walk_ty_alias_where_clauses(vis, where_clauses);
+                        )?
+                        $(<V as Visitor<$lt>>::Result::output())?
+                    }
+                    ItemKind::Enum(ident, generics, enum_definition) => {
+                        try_visit!(vis.visit_ident(ident));
+                        try_visit!(vis.visit_generics(generics));
+                        visit_variants(vis, &$($mut)? enum_definition.variants)
+                    }
+                    ItemKind::Struct(ident, generics, variant_data)
+                    | ItemKind::Union(ident, generics, variant_data) => {
+                        try_visit!(vis.visit_ident(ident));
+                        try_visit!(vis.visit_generics(generics));
+                        vis.visit_variant_data(variant_data)
+                    }
+                    ItemKind::Impl(box Impl {
+                        defaultness,
+                        safety,
+                        generics,
+                        constness,
+                        polarity,
+                        of_trait,
+                        self_ty,
+                        items,
+                    }) => {
+                        try_visit!(visit_defaultness(vis, defaultness));
+                        try_visit!(visit_safety(vis, safety));
+                        try_visit!(vis.visit_generics(generics));
+                        try_visit!(visit_constness(vis, constness));
+                        try_visit!(visit_polarity(vis, polarity));
+                        visit_opt!(vis, visit_trait_ref, of_trait);
+                        try_visit!(vis.visit_ty(self_ty));
+                        visit_assoc_items(vis, items, AssocCtxt::Impl { of_trait: of_trait.is_some() })
+                    }
+                    ItemKind::Trait(box Trait { safety, is_auto: _, ident, generics, bounds, items }) => {
+                        try_visit!(visit_safety(vis, safety));
+                        try_visit!(vis.visit_ident(ident));
+                        try_visit!(vis.visit_generics(generics));
+                        try_visit!(visit_bounds(vis, bounds, BoundKind::Bound));
+                        visit_assoc_items(vis, items, AssocCtxt::Trait)
+                    }
+                    ItemKind::TraitAlias(ident, generics, bounds) => {
+                        try_visit!(vis.visit_ident(ident));
+                        try_visit!(vis.visit_generics(generics));
+                        visit_bounds(vis, bounds, BoundKind::Bound)
+                    }
+                    ItemKind::MacCall(m) => vis.visit_mac_call(m),
+                    ItemKind::MacroDef(ident, def) => {
+                        try_visit!(vis.visit_ident(ident));
+                        vis.visit_macro_def(def)
+                    }
+                    ItemKind::Delegation(box Delegation {
+                        id,
+                        qself,
+                        path,
+                        ident,
+                        rename,
+                        body,
+                        from_glob: _,
+                    }) => {
+                        try_visit!(visit_id(vis, id));
+                        try_visit!(vis.visit_qself(qself));
+                        try_visit!(vis.visit_path(path));
+                        try_visit!(vis.visit_ident(ident));
+                        visit_opt!(vis, visit_ident, rename);
+                        visit_opt!(vis, visit_block, body);
+                        $(<V as Visitor<$lt>>::Result::output())?
+                    }
+                    ItemKind::DelegationMac(box DelegationMac { qself, prefix, suffixes, body }) => {
+                        try_visit!(vis.visit_qself(qself));
+                        try_visit!(vis.visit_path(prefix));
+                        if let Some(suffixes) = suffixes {
+                            for (ident, rename) in suffixes {
+                                try_visit!(vis.visit_ident(ident));
+                                visit_opt!(vis, visit_ident, rename);
+                            }
+                        }
+                        visit_opt!(vis, visit_block, body);
+                        $(<V as Visitor<$lt>>::Result::output())?
+                    }
+                }
             }
-            ItemKind::Fn(func) => {
-                let kind = FnKind::Fn(FnCtxt::Free, vis, &*func);
-                try_visit!(visitor.visit_fn(kind, span, id));
+        }
+
+        fn walk_const_item<$($lt,)? V: $Visitor$(<$lt>)?>(
+            vis: &mut V,
+            item: &$($lt)? $($mut)? ConstItem,
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            let ConstItem { defaultness, ident, generics, ty, expr, define_opaque } = item;
+            try_visit!(visit_defaultness(vis, defaultness));
+            try_visit!(vis.visit_ident(ident));
+            try_visit!(vis.visit_generics(generics));
+            try_visit!(vis.visit_ty(ty));
+            visit_opt!(vis, visit_expr, expr);
+            walk_define_opaques(vis, define_opaque)
+        }
+
+        fn walk_foreign_mod<$($lt,)? V: $Visitor$(<$lt>)?>(vis: &mut V, foreign_mod: &$($lt)? $($mut)? ForeignMod) $(-> <V as Visitor<$lt>>::Result)? {
+            let ForeignMod { extern_span: _, safety, abi: _, items } = foreign_mod;
+            try_visit!(visit_safety(vis, safety));
+            visit_foreign_items(vis, items)
+        }
+
+        fn walk_define_opaques<$($lt,)? V: $Visitor$(<$lt>)?>(
+            visitor: &mut V,
+            define_opaque: &$($lt)? $($mut)? Option<ThinVec<(NodeId, Path)>>,
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            if let Some(define_opaque) = define_opaque {
+                for (id, path) in define_opaque {
+                    try_visit!(visit_id(visitor, id));
+                    try_visit!(visitor.visit_path(path));
+                }
             }
-            ItemKind::Mod(_unsafety, ident, mod_kind) => {
-                try_visit!(visitor.visit_ident(ident));
-                match mod_kind {
-                    ModKind::Loaded(items, _inline, _inner_span, _) => {
-                        walk_list!(visitor, visit_item, items);
+            $(<V as Visitor<$lt>>::Result::output())?
+        }
+
+        impl WalkItemKind for AssocItemKind {
+            type Ctxt = AssocCtxt;
+            fn walk<$($lt,)? V: $Visitor$(<$lt>)?>(
+                &$($lt)? $($mut)? self,
+                span: Span,
+                id: NodeId,
+                visibility: &$($lt)? $($mut)? Visibility,
+                ctxt: Self::Ctxt,
+                vis: &mut V,
+            ) $(-> <V as Visitor<$lt>>::Result)? {
+                match self {
+                    AssocItemKind::Const(item) => {
+                        walk_const_item(vis, item)
+                    }
+                    AssocItemKind::Fn(func) => {
+                        vis.visit_fn(FnKind::Fn(FnCtxt::Assoc(ctxt), visibility, &$($mut)? *func), span, id)
+                    }
+                    AssocItemKind::Type(box TyAlias {
+                        generics,
+                        ident,
+                        bounds,
+                        ty,
+                        defaultness,
+                        $(${ignore($lt)} #[expect(unused)])?
+                        where_clauses,
+                    }) => {
+                        try_visit!(visit_defaultness(vis, defaultness));
+                        try_visit!(vis.visit_ident(ident));
+                        try_visit!(vis.visit_generics(generics));
+                        try_visit!(visit_bounds(vis, bounds, BoundKind::Bound));
+                        visit_opt!(vis, visit_ty, ty);
+                        $(${ignore($mut)}
+                            walk_ty_alias_where_clauses(vis, where_clauses);
+                        )?
+                        $(<V as Visitor<$lt>>::Result::output())?
+                    }
+                    AssocItemKind::MacCall(mac) => {
+                        vis.visit_mac_call(mac)
+                    }
+                    AssocItemKind::Delegation(box Delegation {
+                        id,
+                        qself,
+                        path,
+                        ident,
+                        rename,
+                        body,
+                        from_glob: _,
+                    }) => {
+                        try_visit!(visit_id(vis, id));
+                        try_visit!(vis.visit_qself(qself));
+                        try_visit!(vis.visit_path(path));
+                        try_visit!(vis.visit_ident(ident));
+                        visit_opt!(vis, visit_ident, rename);
+                        visit_opt!(vis, visit_block, body);
+                        $(<V as Visitor<$lt>>::Result::output())?
+                    }
+                    AssocItemKind::DelegationMac(box DelegationMac { qself, prefix, suffixes, body }) => {
+                        try_visit!(vis.visit_qself(qself));
+                        try_visit!(vis.visit_path(prefix));
+                        if let Some(suffixes) = suffixes {
+                            for (ident, rename) in suffixes {
+                                try_visit!(vis.visit_ident(ident));
+                                visit_opt!(vis, visit_ident, rename);
+                            }
+                        }
+                        visit_opt!(vis, visit_block, body);
+                        $(<V as Visitor<$lt>>::Result::output())?
                     }
-                    ModKind::Unloaded => {}
                 }
             }
-            ItemKind::ForeignMod(ForeignMod { extern_span: _, safety: _, abi: _, items }) => {
-                walk_list!(visitor, visit_foreign_item, items);
-            }
-            ItemKind::GlobalAsm(asm) => try_visit!(visitor.visit_inline_asm(asm)),
-            ItemKind::TyAlias(box TyAlias {
-                generics,
-                ident,
-                bounds,
-                ty,
-                defaultness: _,
-                where_clauses: _,
-            }) => {
-                try_visit!(visitor.visit_ident(ident));
-                try_visit!(visitor.visit_generics(generics));
-                walk_list!(visitor, visit_param_bound, bounds, BoundKind::Bound);
-                visit_opt!(visitor, visit_ty, ty);
+        }
+
+        impl WalkItemKind for ForeignItemKind {
+            type Ctxt = ();
+            fn walk<$($lt,)? V: $Visitor$(<$lt>)?>(
+                &$($lt)? $($mut)? self,
+                span: Span,
+                id: NodeId,
+                visibility: &$($lt)? $($mut)? Visibility,
+                _ctxt: Self::Ctxt,
+                vis: &mut V,
+            ) $(-> <V as Visitor<$lt>>::Result)? {
+                match self {
+                    ForeignItemKind::Static(box StaticItem {
+                        ident,
+                        ty,
+                        mutability: _,
+                        expr,
+                        safety: _,
+                        define_opaque,
+                    }) => {
+                        try_visit!(vis.visit_ident(ident));
+                        try_visit!(vis.visit_ty(ty));
+                        visit_opt!(vis, visit_expr, expr);
+                        walk_define_opaques(vis, define_opaque)
+                    }
+                    ForeignItemKind::Fn(func) => {
+                        vis.visit_fn(FnKind::Fn(FnCtxt::Foreign, visibility, &$($mut)?*func), span, id)
+                    }
+                    ForeignItemKind::TyAlias(box TyAlias {
+                        defaultness,
+                        ident,
+                        generics,
+                        bounds,
+                        ty,
+                        $(${ignore($lt)} #[expect(unused)])?
+                        where_clauses,
+                    }) => {
+                        try_visit!(visit_defaultness(vis, defaultness));
+                        try_visit!(vis.visit_ident(ident));
+                        try_visit!(vis.visit_generics(generics));
+                        try_visit!(visit_bounds(vis, bounds, BoundKind::Bound));
+                        visit_opt!(vis, visit_ty, ty);
+                        $(${ignore($mut)}
+                            walk_ty_alias_where_clauses(vis, where_clauses);
+                        )?
+                        $(<V as Visitor<$lt>>::Result::output())?
+                    }
+                    ForeignItemKind::MacCall(mac) => {
+                        vis.visit_mac_call(mac)
+                    }
+                }
             }
-            ItemKind::Enum(ident, enum_definition, generics) => {
-                try_visit!(visitor.visit_ident(ident));
-                try_visit!(visitor.visit_generics(generics));
-                try_visit!(visitor.visit_enum_def(enum_definition));
+        }
+
+        fn walk_coroutine_kind<$($lt,)? V: $Visitor$(<$lt>)?>(
+            vis: &mut V,
+            coroutine_kind: &$($lt)? $($mut)? CoroutineKind,
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            let (CoroutineKind::Async { span, closure_id, return_impl_trait_id }
+                | CoroutineKind::Gen { span, closure_id, return_impl_trait_id }
+                | CoroutineKind::AsyncGen { span, closure_id, return_impl_trait_id })
+                = coroutine_kind;
+            try_visit!(visit_id(vis, closure_id));
+            try_visit!(visit_id(vis, return_impl_trait_id));
+            visit_span(vis, span)
+        }
+
+        pub fn walk_pat<$($lt,)? V: $Visitor$(<$lt>)?>(
+            vis: &mut V,
+            pattern: &$($lt)? $($mut)? Pat
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            let Pat { id, kind, span, tokens: _ } = pattern;
+            try_visit!(visit_id(vis, id));
+            match kind {
+                PatKind::Err(_guar) => {}
+                PatKind::Missing | PatKind::Wild | PatKind::Rest | PatKind::Never => {}
+                PatKind::Ident(_bmode, ident, optional_subpattern) => {
+                    try_visit!(vis.visit_ident(ident));
+                    visit_opt!(vis, visit_pat, optional_subpattern);
+                }
+                PatKind::Expr(expression) => try_visit!(vis.visit_expr(expression)),
+                PatKind::TupleStruct(opt_qself, path, elems) => {
+                    try_visit!(vis.visit_qself(opt_qself));
+                    try_visit!(vis.visit_path(path));
+                    walk_list!(vis, visit_pat, elems);
+                }
+                PatKind::Path(opt_qself, path) => {
+                    try_visit!(vis.visit_qself(opt_qself));
+                    try_visit!(vis.visit_path(path))
+                }
+                PatKind::Struct(opt_qself, path, fields, _rest) => {
+                    try_visit!(vis.visit_qself(opt_qself));
+                    try_visit!(vis.visit_path(path));
+                    try_visit!(visit_pat_fields(vis, fields));
+                }
+                PatKind::Box(subpattern) | PatKind::Deref(subpattern) | PatKind::Paren(subpattern) => {
+                    try_visit!(vis.visit_pat(subpattern));
+                }
+                PatKind::Ref(subpattern, _ /*mutbl*/) => {
+                    try_visit!(vis.visit_pat(subpattern));
+                }
+                PatKind::Range(lower_bound, upper_bound, _end) => {
+                    visit_opt!(vis, visit_expr, lower_bound);
+                    visit_opt!(vis, visit_expr, upper_bound);
+                    try_visit!(visit_span(vis, span));
+                }
+                PatKind::Guard(subpattern, guard_condition) => {
+                    try_visit!(vis.visit_pat(subpattern));
+                    try_visit!(vis.visit_expr(guard_condition));
+                }
+                PatKind::Tuple(elems) | PatKind::Slice(elems) | PatKind::Or(elems) => {
+                    walk_list!(vis, visit_pat, elems);
+                }
+                PatKind::MacCall(mac) => try_visit!(vis.visit_mac_call(mac)),
             }
-            ItemKind::Impl(box Impl {
-                defaultness: _,
-                safety: _,
-                generics,
-                constness: _,
-                polarity: _,
-                of_trait,
-                self_ty,
-                items,
-            }) => {
-                try_visit!(visitor.visit_generics(generics));
-                visit_opt!(visitor, visit_trait_ref, of_trait);
-                try_visit!(visitor.visit_ty(self_ty));
-                walk_list!(
-                    visitor,
-                    visit_assoc_item,
-                    items,
-                    AssocCtxt::Impl { of_trait: of_trait.is_some() }
-                );
+            visit_span(vis, span)
+        }
+
+        pub fn walk_anon_const<$($lt,)? V: $Visitor$(<$lt>)?>(
+            vis: &mut V,
+            constant: &$($lt)? $($mut)? AnonConst,
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            let AnonConst { id, value } = constant;
+            try_visit!(visit_id(vis, id));
+            vis.visit_expr(value)
+        }
+
+        pub fn walk_path_segment<$($lt,)? V: $Visitor$(<$lt>)?>(
+            vis: &mut V,
+            segment: &$($lt)? $($mut)? PathSegment,
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            let PathSegment { ident, id, args } = segment;
+            try_visit!(visit_id(vis, id));
+            try_visit!(vis.visit_ident(ident));
+            visit_opt!(vis, visit_generic_args, args);
+            $(<V as Visitor<$lt>>::Result::output())?
+        }
+
+        pub fn walk_block<$($lt,)? V: $Visitor$(<$lt>)?>(
+            vis: &mut V,
+            block: &$($lt)? $($mut)? Block
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            let Block { stmts, id, rules: _, span, tokens: _ } = block;
+            try_visit!(visit_id(vis, id));
+            try_visit!(visit_stmts(vis, stmts));
+            visit_span(vis, span)
+        }
+
+
+        pub fn walk_ty<$($lt,)? V: $Visitor$(<$lt>)?>(
+            vis: &mut V, ty: &$($lt)? $($mut)? Ty
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            let Ty { id, kind, span, tokens: _ } = ty;
+            try_visit!(visit_id(vis, id));
+            match kind {
+                TyKind::Err(_guar) => {}
+                TyKind::Infer | TyKind::ImplicitSelf | TyKind::Dummy | TyKind::Never | TyKind::CVarArgs => {}
+                TyKind::Slice(ty) | TyKind::Paren(ty) => try_visit!(vis.visit_ty(ty)),
+                TyKind::Ptr(MutTy { ty, mutbl: _ }) => try_visit!(vis.visit_ty(ty)),
+                TyKind::Ref(opt_lifetime, MutTy { ty, mutbl: _ })
+                | TyKind::PinnedRef(opt_lifetime, MutTy { ty, mutbl: _ }) => {
+                    // FIXME(fee1-dead) asymmetry
+                    visit_opt!(vis, visit_lifetime, opt_lifetime$(${ignore($lt)}, LifetimeCtxt::Ref)?);
+                    try_visit!(vis.visit_ty(ty));
+                }
+                TyKind::Tup(tuple_element_types) => {
+                    walk_list!(vis, visit_ty, tuple_element_types);
+                }
+                TyKind::BareFn(function_declaration) => {
+                    let BareFnTy { safety, ext: _, generic_params, decl, decl_span } =
+                        &$($mut)? **function_declaration;
+                    visit_safety(vis, safety);
+                    try_visit!(visit_generic_params(vis, generic_params));
+                    try_visit!(vis.visit_fn_decl(decl));
+                    try_visit!(visit_span(vis, decl_span));
+                }
+                TyKind::UnsafeBinder(binder) => {
+                    try_visit!(visit_generic_params(vis, &$($mut)? binder.generic_params));
+                    try_visit!(vis.visit_ty(&$($mut)? binder.inner_ty));
+                }
+                TyKind::Path(maybe_qself, path) => {
+                    try_visit!(vis.visit_qself(maybe_qself));
+                    try_visit!(vis.visit_path(path));
+                }
+                TyKind::Pat(ty, pat) => {
+                    try_visit!(vis.visit_ty(ty));
+                    try_visit!(vis.visit_ty_pat(pat));
+                }
+                TyKind::Array(ty, length) => {
+                    try_visit!(vis.visit_ty(ty));
+                    try_visit!(vis.visit_anon_const(length));
+                }
+                TyKind::TraitObject(bounds, _syntax) => {
+                    walk_list!(vis, visit_param_bound, bounds, BoundKind::TraitObject);
+                }
+                TyKind::ImplTrait(id, bounds) => {
+                    try_visit!(visit_id(vis, id));
+                    walk_list!(vis, visit_param_bound, bounds, BoundKind::Impl);
+                }
+                TyKind::Typeof(expression) => try_visit!(vis.visit_anon_const(expression)),
+
+                TyKind::MacCall(mac) => try_visit!(vis.visit_mac_call(mac)),
             }
-            ItemKind::Struct(ident, struct_definition, generics)
-            | ItemKind::Union(ident, struct_definition, generics) => {
-                try_visit!(visitor.visit_ident(ident));
-                try_visit!(visitor.visit_generics(generics));
-                try_visit!(visitor.visit_variant_data(struct_definition));
+            visit_span(vis, span)
+        }
+
+        pub fn walk_crate<$($lt,)? V: $Visitor$(<$lt>)?>(
+            vis: &mut V,
+            krate: &$($lt)? $($mut)? Crate,
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            let Crate { attrs, items, spans, id, is_placeholder: _ } = krate;
+            try_visit!(visit_id(vis, id));
+            walk_list!(vis, visit_attribute, attrs);
+            try_visit!(visit_items(vis, items));
+            let ModSpans { inner_span, inject_use_span } = spans;
+            try_visit!(visit_span(vis, inner_span));
+            visit_span(vis, inject_use_span)
+        }
+
+        pub fn walk_local<$($lt,)? V: $Visitor$(<$lt>)?>(
+            vis: &mut V,
+            local: &$($lt)? $($mut)? Local,
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            let Local { id, super_, pat, ty, kind, span, colon_sp, attrs, tokens: _ } = local;
+            if let Some(sp) = super_ {
+                try_visit!(visit_span(vis, sp));
             }
-            ItemKind::Trait(box Trait {
-                safety: _,
-                is_auto: _,
-                ident,
-                generics,
-                bounds,
-                items,
-            }) => {
-                try_visit!(visitor.visit_ident(ident));
-                try_visit!(visitor.visit_generics(generics));
-                walk_list!(visitor, visit_param_bound, bounds, BoundKind::SuperTraits);
-                walk_list!(visitor, visit_assoc_item, items, AssocCtxt::Trait);
+            try_visit!(visit_id(vis, id));
+            walk_list!(vis, visit_attribute, attrs);
+            try_visit!(vis.visit_pat(pat));
+            visit_opt!(vis, visit_ty, ty);
+            match kind {
+                LocalKind::Decl => {}
+                LocalKind::Init(init) => {
+                    try_visit!(vis.visit_expr(init))
+                }
+                LocalKind::InitElse(init, els) => {
+                    try_visit!(vis.visit_expr(init));
+                    try_visit!(vis.visit_block(els));
+                }
             }
-            ItemKind::TraitAlias(ident, generics, bounds) => {
-                try_visit!(visitor.visit_ident(ident));
-                try_visit!(visitor.visit_generics(generics));
-                walk_list!(visitor, visit_param_bound, bounds, BoundKind::Bound);
+            if let Some(sp) = colon_sp {
+                try_visit!(visit_span(vis, sp));
             }
-            ItemKind::MacCall(mac) => try_visit!(visitor.visit_mac_call(mac)),
-            ItemKind::MacroDef(ident, ts) => {
-                try_visit!(visitor.visit_ident(ident));
-                try_visit!(visitor.visit_mac_def(ts, id))
+            visit_span(vis, span)
+        }
+
+        pub fn walk_poly_trait_ref<$($lt,)? V: $Visitor$(<$lt>)?>(
+            vis: &mut V,
+            p: &$($lt)? $($mut)? PolyTraitRef,
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            let PolyTraitRef { bound_generic_params, modifiers, trait_ref, span } = p;
+            try_visit!(visit_modifiers(vis, modifiers));
+            try_visit!(visit_generic_params(vis, bound_generic_params));
+            try_visit!(vis.visit_trait_ref(trait_ref));
+            visit_span(vis, span)
+        }
+
+        pub fn walk_trait_ref<$($lt,)? V: $Visitor$(<$lt>)?>(
+            vis: &mut V,
+            TraitRef { path, ref_id }: &$($lt)? $($mut)? TraitRef,
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            try_visit!(vis.visit_path(path));
+            visit_id(vis, ref_id)
+        }
+
+        pub fn walk_variant<$($lt,)? V: $Visitor$(<$lt>)?>(
+            vis: &mut V,
+            variant: &$($lt)? $($mut)? Variant,
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            let Variant { attrs, id, span, vis: visibility, ident, data, disr_expr, is_placeholder: _ } = variant;
+            try_visit!(visit_id(vis, id));
+            walk_list!(vis, visit_attribute, attrs);
+            try_visit!(vis.visit_vis(visibility));
+            try_visit!(vis.visit_ident(ident));
+            try_visit!(vis.visit_variant_data(data));
+            $(${ignore($lt)} visit_opt!(vis, visit_variant_discr, disr_expr); )?
+            $(${ignore($mut)} visit_opt!(vis, visit_anon_const, disr_expr); )?
+            visit_span(vis, span)
+        }
+
+        pub fn walk_expr_field<$($lt,)? V: $Visitor$(<$lt>)?>(
+            vis: &mut V,
+            f: &$($lt)? $($mut)? ExprField,
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            let ExprField { attrs, id, span, ident, expr, is_shorthand: _, is_placeholder: _ } = f;
+            try_visit!(visit_id(vis, id));
+            walk_list!(vis, visit_attribute, attrs);
+            try_visit!(vis.visit_ident(ident));
+            try_visit!(vis.visit_expr(expr));
+            visit_span(vis, span)
+        }
+
+        pub fn walk_pat_field<$($lt,)? V: $Visitor$(<$lt>)?>(
+            vis: &mut V,
+            fp: &$($lt)? $($mut)? PatField,
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            let PatField { ident, pat, is_shorthand: _, attrs, id, span, is_placeholder: _ } = fp;
+            try_visit!(visit_id(vis, id));
+            walk_list!(vis, visit_attribute, attrs);
+            try_visit!(vis.visit_ident(ident));
+            try_visit!(vis.visit_pat(pat));
+            visit_span(vis, span)
+        }
+
+        pub fn walk_ty_pat<$($lt,)? V: $Visitor$(<$lt>)?>(
+            vis: &mut V,
+            tp: &$($lt)? $($mut)? TyPat,
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            let TyPat { id, kind, span, tokens: _ } = tp;
+            try_visit!(visit_id(vis, id));
+            match kind {
+                TyPatKind::Range(start, end, _include_end) => {
+                    visit_opt!(vis, visit_anon_const, start);
+                    visit_opt!(vis, visit_anon_const, end);
+                }
+                TyPatKind::Or(variants) => walk_list!(vis, visit_ty_pat, variants),
+                TyPatKind::Err(_) => {}
             }
-            ItemKind::Delegation(box Delegation {
-                id,
-                qself,
-                path,
-                ident,
-                rename,
-                body,
-                from_glob: _,
-            }) => {
-                try_visit!(visitor.visit_qself(qself));
-                try_visit!(visitor.visit_path(path, *id));
-                try_visit!(visitor.visit_ident(ident));
-                visit_opt!(visitor, visit_ident, rename);
-                visit_opt!(visitor, visit_block, body);
+            visit_span(vis, span)
+        }
+
+        fn walk_qself<$($lt,)? V: $Visitor$(<$lt>)?>(
+            vis: &mut V,
+            qself: &$($lt)? $($mut)? Option<P<QSelf>>,
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            if let Some(qself) = qself {
+                let QSelf { ty, path_span, position: _ } = &$($mut)? **qself;
+                try_visit!(vis.visit_ty(ty));
+                try_visit!(visit_span(vis, path_span));
             }
-            ItemKind::DelegationMac(box DelegationMac { qself, prefix, suffixes, body }) => {
-                try_visit!(visitor.visit_qself(qself));
-                try_visit!(visitor.visit_path(prefix, id));
-                if let Some(suffixes) = suffixes {
-                    for (ident, rename) in suffixes {
-                        visitor.visit_ident(ident);
-                        if let Some(rename) = rename {
-                            visitor.visit_ident(rename);
-                        }
+            $(<V as Visitor<$lt>>::Result::output())?
+        }
+
+        pub fn walk_path<$($lt,)? V: $Visitor$(<$lt>)?>(
+            vis: &mut V,
+            path: &$($lt)? $($mut)? Path,
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            let Path { span, segments, tokens: _ } = path;
+            walk_list!(vis, visit_path_segment, segments);
+            visit_span(vis, span)
+        }
+
+        pub fn walk_use_tree<$($lt,)? V: $Visitor$(<$lt>)?>(
+            vis: &mut V,
+            use_tree: &$($lt)? $($mut)? UseTree,
+        ) $(-> <V as Visitor<$lt>>::Result)? {
+            let UseTree { prefix, kind, span } = use_tree;
+            try_visit!(vis.visit_path(prefix));
+            match kind {
+                UseTreeKind::Simple(rename) => {
+                    // The extra IDs are handled during AST lowering.
+                    visit_opt!(vis, visit_ident, rename);
+                }
+                UseTreeKind::Glob => {}
+                UseTreeKind::Nested { items, span } => {
+                    for (nested_tree, nested_id) in items {
+                        try_visit!(visit_nested_use_tree(vis, nested_tree, nested_id));
                     }
+                    try_visit!(visit_span(vis, span));
                 }
-                visit_opt!(visitor, visit_block, body);
             }
+            visit_span(vis, span)
         }
-        V::Result::output()
-    }
-}
-
-pub fn walk_enum_def<'a, V: Visitor<'a>>(
-    visitor: &mut V,
-    EnumDef { variants }: &'a EnumDef,
-) -> V::Result {
-    walk_list!(visitor, visit_variant, variants);
-    V::Result::output()
-}
-
-pub fn walk_variant<'a, V: Visitor<'a>>(visitor: &mut V, variant: &'a Variant) -> V::Result
-where
-    V: Visitor<'a>,
-{
-    let Variant { attrs, id: _, span: _, vis, ident, data, disr_expr, is_placeholder: _ } = variant;
-    walk_list!(visitor, visit_attribute, attrs);
-    try_visit!(visitor.visit_vis(vis));
-    try_visit!(visitor.visit_ident(ident));
-    try_visit!(visitor.visit_variant_data(data));
-    visit_opt!(visitor, visit_variant_discr, disr_expr);
-    V::Result::output()
-}
-
-pub fn walk_expr_field<'a, V: Visitor<'a>>(visitor: &mut V, f: &'a ExprField) -> V::Result {
-    let ExprField { attrs, id: _, span: _, ident, expr, is_shorthand: _, is_placeholder: _ } = f;
-    walk_list!(visitor, visit_attribute, attrs);
-    try_visit!(visitor.visit_ident(ident));
-    try_visit!(visitor.visit_expr(expr));
-    V::Result::output()
-}
-
-pub fn walk_pat_field<'a, V: Visitor<'a>>(visitor: &mut V, fp: &'a PatField) -> V::Result {
-    let PatField { ident, pat, is_shorthand: _, attrs, id: _, span: _, is_placeholder: _ } = fp;
-    walk_list!(visitor, visit_attribute, attrs);
-    try_visit!(visitor.visit_ident(ident));
-    try_visit!(visitor.visit_pat(pat));
-    V::Result::output()
-}
-
-pub fn walk_ty<'a, V: Visitor<'a>>(visitor: &mut V, typ: &'a Ty) -> V::Result {
-    let Ty { id, kind, span: _, tokens: _ } = typ;
-    match kind {
-        TyKind::Slice(ty) | TyKind::Paren(ty) => try_visit!(visitor.visit_ty(ty)),
-        TyKind::Ptr(MutTy { ty, mutbl: _ }) => try_visit!(visitor.visit_ty(ty)),
-        TyKind::Ref(opt_lifetime, MutTy { ty, mutbl: _ })
-        | TyKind::PinnedRef(opt_lifetime, MutTy { ty, mutbl: _ }) => {
-            visit_opt!(visitor, visit_lifetime, opt_lifetime, LifetimeCtxt::Ref);
-            try_visit!(visitor.visit_ty(ty));
-        }
-        TyKind::Tup(tuple_element_types) => {
-            walk_list!(visitor, visit_ty, tuple_element_types);
-        }
-        TyKind::BareFn(function_declaration) => {
-            let BareFnTy { safety: _, ext: _, generic_params, decl, decl_span: _ } =
-                &**function_declaration;
-            walk_list!(visitor, visit_generic_param, generic_params);
-            try_visit!(visitor.visit_fn_decl(decl));
-        }
-        TyKind::UnsafeBinder(binder) => {
-            walk_list!(visitor, visit_generic_param, &binder.generic_params);
-            try_visit!(visitor.visit_ty(&binder.inner_ty));
-        }
-        TyKind::Path(maybe_qself, path) => {
-            try_visit!(visitor.visit_qself(maybe_qself));
-            try_visit!(visitor.visit_path(path, *id));
-        }
-        TyKind::Pat(ty, pat) => {
-            try_visit!(visitor.visit_ty(ty));
-            try_visit!(visitor.visit_ty_pat(pat));
-        }
-        TyKind::Array(ty, length) => {
-            try_visit!(visitor.visit_ty(ty));
-            try_visit!(visitor.visit_anon_const(length));
-        }
-        TyKind::TraitObject(bounds, _syntax) => {
-            walk_list!(visitor, visit_param_bound, bounds, BoundKind::TraitObject);
-        }
-        TyKind::ImplTrait(_id, bounds) => {
-            walk_list!(visitor, visit_param_bound, bounds, BoundKind::Impl);
-        }
-        TyKind::Typeof(expression) => try_visit!(visitor.visit_anon_const(expression)),
-        TyKind::Infer | TyKind::ImplicitSelf | TyKind::Dummy => {}
-        TyKind::Err(_guar) => {}
-        TyKind::MacCall(mac) => try_visit!(visitor.visit_mac_call(mac)),
-        TyKind::Never | TyKind::CVarArgs => {}
-    }
-    V::Result::output()
+    };
 }
 
-pub fn walk_ty_pat<'a, V: Visitor<'a>>(visitor: &mut V, tp: &'a TyPat) -> V::Result {
-    let TyPat { id: _, kind, span: _, tokens: _ } = tp;
-    match kind {
-        TyPatKind::Range(start, end, _include_end) => {
-            visit_opt!(visitor, visit_anon_const, start);
-            visit_opt!(visitor, visit_anon_const, end);
-        }
-        TyPatKind::Or(variants) => walk_list!(visitor, visit_ty_pat, variants),
-        TyPatKind::Err(_) => {}
-    }
-    V::Result::output()
-}
+common_visitor_and_walkers!(Visitor<'a>);
 
-fn walk_qself<'a, V: Visitor<'a>>(visitor: &mut V, qself: &'a Option<P<QSelf>>) -> V::Result {
-    if let Some(qself) = qself {
-        let QSelf { ty, path_span: _, position: _ } = &**qself;
-        try_visit!(visitor.visit_ty(ty));
+macro_rules! generate_list_visit_fns {
+    ($($name:ident, $Ty:ty, $visit_fn:ident$(, $param:ident: $ParamTy:ty)*;)+) => {
+        $(
+            fn $name<'a, V: Visitor<'a>>(
+                vis: &mut V,
+                values: &'a ThinVec<$Ty>,
+                $(
+                    $param: $ParamTy,
+                )*
+            ) -> V::Result {
+                walk_list!(vis, $visit_fn, values$(,$param)*);
+                V::Result::output()
+            }
+        )+
     }
-    V::Result::output()
 }
 
-pub fn walk_path<'a, V: Visitor<'a>>(visitor: &mut V, path: &'a Path) -> V::Result {
-    let Path { span: _, segments, tokens: _ } = path;
-    walk_list!(visitor, visit_path_segment, segments);
-    V::Result::output()
-}
-
-pub fn walk_use_tree<'a, V: Visitor<'a>>(
-    visitor: &mut V,
-    use_tree: &'a UseTree,
-    id: NodeId,
-) -> V::Result {
-    let UseTree { prefix, kind, span: _ } = use_tree;
-    try_visit!(visitor.visit_path(prefix, id));
-    match kind {
-        UseTreeKind::Simple(rename) => {
-            // The extra IDs are handled during AST lowering.
-            visit_opt!(visitor, visit_ident, rename);
-        }
-        UseTreeKind::Glob => {}
-        UseTreeKind::Nested { items, span: _ } => {
-            for &(ref nested_tree, nested_id) in items {
-                try_visit!(visitor.visit_use_tree(nested_tree, nested_id, true));
-            }
-        }
-    }
-    V::Result::output()
+generate_list_visit_fns! {
+    visit_items, P<Item>, visit_item;
+    visit_foreign_items, P<ForeignItem>, visit_foreign_item;
+    visit_generic_params, GenericParam, visit_generic_param;
+    visit_stmts, Stmt, visit_stmt;
+    visit_pat_fields, PatField, visit_pat_field;
+    visit_variants, Variant, visit_variant;
+    visit_assoc_items, P<AssocItem>, visit_assoc_item, ctxt: AssocCtxt;
 }
 
-pub fn walk_path_segment<'a, V: Visitor<'a>>(
-    visitor: &mut V,
-    segment: &'a PathSegment,
+#[expect(rustc::pass_by_value)] // needed for symmetry with mut_visit
+fn visit_nested_use_tree<'a, V: Visitor<'a>>(
+    vis: &mut V,
+    nested_tree: &'a UseTree,
+    &nested_id: &NodeId,
 ) -> V::Result {
-    let PathSegment { ident, id: _, args } = segment;
-    try_visit!(visitor.visit_ident(ident));
-    visit_opt!(visitor, visit_generic_args, args);
-    V::Result::output()
+    vis.visit_nested_use_tree(nested_tree, nested_id)
 }
 
 pub fn walk_generic_args<'a, V>(visitor: &mut V, generic_args: &'a GenericArgs) -> V::Result
@@ -775,101 +1171,6 @@ pub fn walk_assoc_item_constraint<'a, V: Visitor<'a>>(
     V::Result::output()
 }
 
-pub fn walk_pat<'a, V: Visitor<'a>>(visitor: &mut V, pattern: &'a Pat) -> V::Result {
-    let Pat { id, kind, span: _, tokens: _ } = pattern;
-    match kind {
-        PatKind::TupleStruct(opt_qself, path, elems) => {
-            try_visit!(visitor.visit_qself(opt_qself));
-            try_visit!(visitor.visit_path(path, *id));
-            walk_list!(visitor, visit_pat, elems);
-        }
-        PatKind::Path(opt_qself, path) => {
-            try_visit!(visitor.visit_qself(opt_qself));
-            try_visit!(visitor.visit_path(path, *id))
-        }
-        PatKind::Struct(opt_qself, path, fields, _rest) => {
-            try_visit!(visitor.visit_qself(opt_qself));
-            try_visit!(visitor.visit_path(path, *id));
-            walk_list!(visitor, visit_pat_field, fields);
-        }
-        PatKind::Box(subpattern) | PatKind::Deref(subpattern) | PatKind::Paren(subpattern) => {
-            try_visit!(visitor.visit_pat(subpattern));
-        }
-        PatKind::Ref(subpattern, _ /*mutbl*/) => {
-            try_visit!(visitor.visit_pat(subpattern));
-        }
-        PatKind::Ident(_bmode, ident, optional_subpattern) => {
-            try_visit!(visitor.visit_ident(ident));
-            visit_opt!(visitor, visit_pat, optional_subpattern);
-        }
-        PatKind::Expr(expression) => try_visit!(visitor.visit_expr(expression)),
-        PatKind::Range(lower_bound, upper_bound, _end) => {
-            visit_opt!(visitor, visit_expr, lower_bound);
-            visit_opt!(visitor, visit_expr, upper_bound);
-        }
-        PatKind::Guard(subpattern, guard_condition) => {
-            try_visit!(visitor.visit_pat(subpattern));
-            try_visit!(visitor.visit_expr(guard_condition));
-        }
-        PatKind::Missing | PatKind::Wild | PatKind::Rest | PatKind::Never => {}
-        PatKind::Err(_guar) => {}
-        PatKind::Tuple(elems) | PatKind::Slice(elems) | PatKind::Or(elems) => {
-            walk_list!(visitor, visit_pat, elems);
-        }
-        PatKind::MacCall(mac) => try_visit!(visitor.visit_mac_call(mac)),
-    }
-    V::Result::output()
-}
-
-impl WalkItemKind for ForeignItemKind {
-    type Ctxt = ();
-    fn walk<'a, V: Visitor<'a>>(
-        &'a self,
-        span: Span,
-        id: NodeId,
-        vis: &'a Visibility,
-        _ctxt: Self::Ctxt,
-        visitor: &mut V,
-    ) -> V::Result {
-        match self {
-            ForeignItemKind::Static(box StaticItem {
-                ident,
-                ty,
-                mutability: _,
-                expr,
-                safety: _,
-                define_opaque,
-            }) => {
-                try_visit!(visitor.visit_ident(ident));
-                try_visit!(visitor.visit_ty(ty));
-                visit_opt!(visitor, visit_expr, expr);
-                try_visit!(walk_define_opaques(visitor, define_opaque));
-            }
-            ForeignItemKind::Fn(func) => {
-                let kind = FnKind::Fn(FnCtxt::Foreign, vis, &*func);
-                try_visit!(visitor.visit_fn(kind, span, id));
-            }
-            ForeignItemKind::TyAlias(box TyAlias {
-                generics,
-                ident,
-                bounds,
-                ty,
-                defaultness: _,
-                where_clauses: _,
-            }) => {
-                try_visit!(visitor.visit_ident(ident));
-                try_visit!(visitor.visit_generics(generics));
-                walk_list!(visitor, visit_param_bound, bounds, BoundKind::Bound);
-                visit_opt!(visitor, visit_ty, ty);
-            }
-            ForeignItemKind::MacCall(mac) => {
-                try_visit!(visitor.visit_mac_call(mac));
-            }
-        }
-        V::Result::output()
-    }
-}
-
 pub fn walk_param_bound<'a, V: Visitor<'a>>(visitor: &mut V, bound: &'a GenericBound) -> V::Result {
     match bound {
         GenericBound::Trait(trait_ref) => visitor.visit_poly_trait_ref(trait_ref),
@@ -887,7 +1188,10 @@ pub fn walk_precise_capturing_arg<'a, V: Visitor<'a>>(
 ) -> V::Result {
     match arg {
         PreciseCapturingArg::Lifetime(lt) => visitor.visit_lifetime(lt, LifetimeCtxt::GenericArg),
-        PreciseCapturingArg::Arg(path, id) => visitor.visit_path(path, *id),
+        PreciseCapturingArg::Arg(path, id) => {
+            try_visit!(visitor.visit_id(*id));
+            visitor.visit_path(path)
+        }
     }
 }
 
@@ -1028,116 +1332,9 @@ pub fn walk_fn<'a, V: Visitor<'a>>(visitor: &mut V, kind: FnKind<'a>) -> V::Resu
     V::Result::output()
 }
 
-impl WalkItemKind for AssocItemKind {
-    type Ctxt = AssocCtxt;
-    fn walk<'a, V: Visitor<'a>>(
-        &'a self,
-        span: Span,
-        id: NodeId,
-        vis: &'a Visibility,
-        ctxt: Self::Ctxt,
-        visitor: &mut V,
-    ) -> V::Result {
-        match self {
-            AssocItemKind::Const(box ConstItem {
-                defaultness: _,
-                ident,
-                generics,
-                ty,
-                expr,
-                define_opaque,
-            }) => {
-                try_visit!(visitor.visit_ident(ident));
-                try_visit!(visitor.visit_generics(generics));
-                try_visit!(visitor.visit_ty(ty));
-                visit_opt!(visitor, visit_expr, expr);
-                try_visit!(walk_define_opaques(visitor, define_opaque));
-            }
-            AssocItemKind::Fn(func) => {
-                let kind = FnKind::Fn(FnCtxt::Assoc(ctxt), vis, &*func);
-                try_visit!(visitor.visit_fn(kind, span, id));
-            }
-            AssocItemKind::Type(box TyAlias {
-                generics,
-                ident,
-                bounds,
-                ty,
-                defaultness: _,
-                where_clauses: _,
-            }) => {
-                try_visit!(visitor.visit_generics(generics));
-                try_visit!(visitor.visit_ident(ident));
-                walk_list!(visitor, visit_param_bound, bounds, BoundKind::Bound);
-                visit_opt!(visitor, visit_ty, ty);
-            }
-            AssocItemKind::MacCall(mac) => {
-                try_visit!(visitor.visit_mac_call(mac));
-            }
-            AssocItemKind::Delegation(box Delegation {
-                id,
-                qself,
-                path,
-                ident,
-                rename,
-                body,
-                from_glob: _,
-            }) => {
-                try_visit!(visitor.visit_qself(qself));
-                try_visit!(visitor.visit_path(path, *id));
-                try_visit!(visitor.visit_ident(ident));
-                visit_opt!(visitor, visit_ident, rename);
-                visit_opt!(visitor, visit_block, body);
-            }
-            AssocItemKind::DelegationMac(box DelegationMac { qself, prefix, suffixes, body }) => {
-                try_visit!(visitor.visit_qself(qself));
-                try_visit!(visitor.visit_path(prefix, id));
-                if let Some(suffixes) = suffixes {
-                    for (ident, rename) in suffixes {
-                        visitor.visit_ident(ident);
-                        if let Some(rename) = rename {
-                            visitor.visit_ident(rename);
-                        }
-                    }
-                }
-                visit_opt!(visitor, visit_block, body);
-            }
-        }
-        V::Result::output()
-    }
-}
-
-pub fn walk_item<'a, V: Visitor<'a>>(
-    visitor: &mut V,
-    item: &'a Item<impl WalkItemKind<Ctxt = ()>>,
-) -> V::Result {
-    walk_item_ctxt(visitor, item, ())
-}
-
-pub fn walk_assoc_item<'a, V: Visitor<'a>>(
-    visitor: &mut V,
-    item: &'a AssocItem,
-    ctxt: AssocCtxt,
-) -> V::Result {
-    walk_item_ctxt(visitor, item, ctxt)
-}
-
-fn walk_item_ctxt<'a, V: Visitor<'a>, K: WalkItemKind>(
-    visitor: &mut V,
-    item: &'a Item<K>,
-    ctxt: K::Ctxt,
-) -> V::Result {
-    let Item { id, span, vis, attrs, kind, tokens: _ } = item;
-    walk_list!(visitor, visit_attribute, attrs);
-    try_visit!(visitor.visit_vis(vis));
-    try_visit!(kind.walk(*span, *id, vis, ctxt, visitor));
-    V::Result::output()
-}
-
-pub fn walk_struct_def<'a, V: Visitor<'a>>(
-    visitor: &mut V,
-    struct_definition: &'a VariantData,
-) -> V::Result {
-    walk_list!(visitor, visit_field_def, struct_definition.fields());
+pub fn walk_variant_data<'a, V: Visitor<'a>>(visitor: &mut V, data: &'a VariantData) -> V::Result {
+    visit_opt!(visitor, visit_id, data.ctor_node_id());
+    walk_list!(visitor, visit_field_def, data.fields());
     V::Result::output()
 }
 
@@ -1152,12 +1349,6 @@ pub fn walk_field_def<'a, V: Visitor<'a>>(visitor: &mut V, field: &'a FieldDef)
     V::Result::output()
 }
 
-pub fn walk_block<'a, V: Visitor<'a>>(visitor: &mut V, block: &'a Block) -> V::Result {
-    let Block { stmts, id: _, rules: _, span: _, tokens: _ } = block;
-    walk_list!(visitor, visit_stmt, stmts);
-    V::Result::output()
-}
-
 pub fn walk_stmt<'a, V: Visitor<'a>>(visitor: &mut V, statement: &'a Stmt) -> V::Result {
     let Stmt { id: _, kind, span: _ } = statement;
     match kind {
@@ -1176,12 +1367,7 @@ pub fn walk_stmt<'a, V: Visitor<'a>>(visitor: &mut V, statement: &'a Stmt) -> V:
 
 pub fn walk_mac<'a, V: Visitor<'a>>(visitor: &mut V, mac: &'a MacCall) -> V::Result {
     let MacCall { path, args: _ } = mac;
-    visitor.visit_path(path, DUMMY_NODE_ID)
-}
-
-pub fn walk_anon_const<'a, V: Visitor<'a>>(visitor: &mut V, constant: &'a AnonConst) -> V::Result {
-    let AnonConst { id: _, value } = constant;
-    visitor.visit_expr(value)
+    visitor.visit_path(path)
 }
 
 pub fn walk_inline_asm<'a, V: Visitor<'a>>(visitor: &mut V, asm: &'a InlineAsm) -> V::Result {
@@ -1221,7 +1407,8 @@ pub fn walk_inline_asm_sym<'a, V: Visitor<'a>>(
     InlineAsmSym { id, qself, path }: &'a InlineAsmSym,
 ) -> V::Result {
     try_visit!(visitor.visit_qself(qself));
-    visitor.visit_path(path, *id)
+    try_visit!(visitor.visit_id(*id));
+    visitor.visit_path(path)
 }
 
 pub fn walk_format_args<'a, V: Visitor<'a>>(visitor: &mut V, fmt: &'a FormatArgs) -> V::Result {
@@ -1253,7 +1440,8 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) -> V
         ExprKind::Struct(se) => {
             let StructExpr { qself, path, fields, rest } = &**se;
             try_visit!(visitor.visit_qself(qself));
-            try_visit!(visitor.visit_path(path, *id));
+            try_visit!(visitor.visit_id(*id));
+            try_visit!(visitor.visit_path(path));
             walk_list!(visitor, visit_expr_field, fields);
             match rest {
                 StructRest::Base(expr) => try_visit!(visitor.visit_expr(expr)),
@@ -1363,7 +1551,8 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) -> V
         ExprKind::Underscore => {}
         ExprKind::Path(maybe_qself, path) => {
             try_visit!(visitor.visit_qself(maybe_qself));
-            try_visit!(visitor.visit_path(path, *id));
+            try_visit!(visitor.visit_id(*id));
+            try_visit!(visitor.visit_path(path));
         }
         ExprKind::Break(opt_label, opt_expr) => {
             visit_opt!(visitor, visit_label, opt_label);
@@ -1426,7 +1615,8 @@ pub fn walk_vis<'a, V: Visitor<'a>>(visitor: &mut V, vis: &'a Visibility) -> V::
     let Visibility { kind, span: _, tokens: _ } = vis;
     match kind {
         VisibilityKind::Restricted { path, id, shorthand: _ } => {
-            try_visit!(visitor.visit_path(path, *id));
+            try_visit!(visitor.visit_id(*id));
+            try_visit!(visitor.visit_path(path));
         }
         VisibilityKind::Public | VisibilityKind::Inherited => {}
     }
@@ -1439,7 +1629,7 @@ pub fn walk_attribute<'a, V: Visitor<'a>>(visitor: &mut V, attr: &'a Attribute)
         AttrKind::Normal(normal) => {
             let NormalAttr { item, tokens: _ } = &**normal;
             let AttrItem { unsafety: _, path, args, tokens: _ } = item;
-            try_visit!(visitor.visit_path(path, DUMMY_NODE_ID));
+            try_visit!(visitor.visit_path(path));
             try_visit!(walk_attr_args(visitor, args));
         }
         AttrKind::DocComment(_kind, _sym) => {}
@@ -1455,15 +1645,3 @@ pub fn walk_attr_args<'a, V: Visitor<'a>>(visitor: &mut V, args: &'a AttrArgs) -
     }
     V::Result::output()
 }
-
-fn walk_define_opaques<'a, V: Visitor<'a>>(
-    visitor: &mut V,
-    define_opaque: &'a Option<ThinVec<(NodeId, Path)>>,
-) -> V::Result {
-    if let Some(define_opaque) = define_opaque {
-        for (id, path) in define_opaque {
-            try_visit!(visitor.visit_path(path, *id));
-        }
-    }
-    V::Result::output()
-}
diff --git a/compiler/rustc_ast_lowering/src/delegation.rs b/compiler/rustc_ast_lowering/src/delegation.rs
index 93c627f64c9..42d25b512f5 100644
--- a/compiler/rustc_ast_lowering/src/delegation.rs
+++ b/compiler/rustc_ast_lowering/src/delegation.rs
@@ -446,13 +446,7 @@ impl<'a> SelfResolver<'a> {
 }
 
 impl<'ast, 'a> Visitor<'ast> for SelfResolver<'a> {
-    fn visit_path(&mut self, path: &'ast Path, id: NodeId) {
+    fn visit_id(&mut self, id: NodeId) {
         self.try_replace_id(id);
-        visit::walk_path(self, path);
-    }
-
-    fn visit_path_segment(&mut self, seg: &'ast PathSegment) {
-        self.try_replace_id(seg.id);
-        visit::walk_path_segment(self, seg);
     }
 }
diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index 9f3aed9216c..537d4a2a6af 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -1,4 +1,3 @@
-use std::assert_matches::assert_matches;
 use std::ops::ControlFlow;
 use std::sync::Arc;
 
@@ -1199,11 +1198,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let closure_def_id = self.local_def_id(closure_id);
         let (binder_clause, generic_params) = self.lower_closure_binder(binder);
 
-        assert_matches!(
-            coroutine_kind,
-            CoroutineKind::Async { .. },
-            "only async closures are supported currently"
-        );
+        let coroutine_desugaring = match coroutine_kind {
+            CoroutineKind::Async { .. } => hir::CoroutineDesugaring::Async,
+            CoroutineKind::Gen { .. } => hir::CoroutineDesugaring::Gen,
+            CoroutineKind::AsyncGen { span, .. } => {
+                span_bug!(span, "only async closures and `iter!` closures are supported currently")
+            }
+        };
 
         let body = self.with_new_scopes(fn_decl_span, |this| {
             let inner_decl =
@@ -1247,7 +1248,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             // Lower this as a `CoroutineClosure`. That will ensure that HIR typeck
             // knows that a `FnDecl` output type like `-> &str` actually means
             // "coroutine that returns &str", rather than directly returning a `&str`.
-            kind: hir::ClosureKind::CoroutineClosure(hir::CoroutineDesugaring::Async),
+            kind: hir::ClosureKind::CoroutineClosure(coroutine_desugaring),
             constness: hir::Constness::NotConst,
         });
         hir::ExprKind::Closure(c)
diff --git a/compiler/rustc_ast_lowering/src/format.rs b/compiler/rustc_ast_lowering/src/format.rs
index 0de0319c667..17b443b8ecc 100644
--- a/compiler/rustc_ast_lowering/src/format.rs
+++ b/compiler/rustc_ast_lowering/src/format.rs
@@ -6,7 +6,7 @@ use rustc_ast::*;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_hir as hir;
 use rustc_session::config::FmtDebug;
-use rustc_span::{Ident, Span, Symbol, kw, sym};
+use rustc_span::{Ident, Span, Symbol, sym};
 
 use super::LoweringContext;
 
@@ -418,7 +418,7 @@ fn expand_format_args<'hir>(
                 &FormatArgsPiece::Placeholder(_) => {
                     // Inject empty string before placeholders when not already preceded by a literal piece.
                     if i == 0 || matches!(fmt.template[i - 1], FormatArgsPiece::Placeholder(_)) {
-                        Some(ctx.expr_str(fmt.span, kw::Empty))
+                        Some(ctx.expr_str(fmt.span, sym::empty))
                     } else {
                         None
                     }
diff --git a/compiler/rustc_ast_lowering/src/index.rs b/compiler/rustc_ast_lowering/src/index.rs
index 26c0e7e5f82..956cb580d10 100644
--- a/compiler/rustc_ast_lowering/src/index.rs
+++ b/compiler/rustc_ast_lowering/src/index.rs
@@ -164,7 +164,7 @@ impl<'a, 'hir> Visitor<'hir> for NodeCollector<'a, 'hir> {
     fn visit_item(&mut self, i: &'hir Item<'hir>) {
         debug_assert_eq!(i.owner_id, self.owner);
         self.with_parent(i.hir_id(), |this| {
-            if let ItemKind::Struct(_, struct_def, _) = &i.kind {
+            if let ItemKind::Struct(_, _, struct_def) = &i.kind {
                 // If this is a tuple or unit-like struct, register the constructor.
                 if let Some(ctor_hir_id) = struct_def.ctor_hir_id() {
                     this.insert(i.span, ctor_hir_id, Node::Ctor(struct_def));
diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs
index f48a571b86a..d1a2ddbdb34 100644
--- a/compiler/rustc_ast_lowering/src/item.rs
+++ b/compiler/rustc_ast_lowering/src/item.rs
@@ -3,10 +3,11 @@ use rustc_ast::ptr::P;
 use rustc_ast::visit::AssocCtxt;
 use rustc_ast::*;
 use rustc_errors::ErrorGuaranteed;
-use rustc_hir::def::{DefKind, Res};
+use rustc_hir::def::{DefKind, PerNS, Res};
 use rustc_hir::def_id::{CRATE_DEF_ID, LocalDefId};
 use rustc_hir::{self as hir, HirId, LifetimeSource, PredicateOrigin};
 use rustc_index::{IndexSlice, IndexVec};
+use rustc_middle::span_bug;
 use rustc_middle::ty::{ResolverAstLowering, TyCtxt};
 use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::{DUMMY_SP, DesugaringKind, Ident, Span, Symbol, kw, sym};
@@ -70,44 +71,32 @@ impl<'a, 'hir> ItemLowerer<'a, 'hir> {
         }
     }
 
-    pub(super) fn lower_node(&mut self, def_id: LocalDefId) -> hir::MaybeOwner<'hir> {
+    pub(super) fn lower_node(&mut self, def_id: LocalDefId) {
         let owner = self.owners.ensure_contains_elem(def_id, || hir::MaybeOwner::Phantom);
         if let hir::MaybeOwner::Phantom = owner {
             let node = self.ast_index[def_id];
             match node {
                 AstOwner::NonOwner => {}
-                AstOwner::Crate(c) => self.lower_crate(c),
-                AstOwner::Item(item) => self.lower_item(item),
-                AstOwner::AssocItem(item, ctxt) => self.lower_assoc_item(item, ctxt),
-                AstOwner::ForeignItem(item) => self.lower_foreign_item(item),
+                AstOwner::Crate(c) => {
+                    debug_assert_eq!(self.resolver.node_id_to_def_id[&CRATE_NODE_ID], CRATE_DEF_ID);
+                    self.with_lctx(CRATE_NODE_ID, |lctx| {
+                        let module = lctx.lower_mod(&c.items, &c.spans);
+                        // FIXME(jdonszelman): is dummy span ever a problem here?
+                        lctx.lower_attrs(hir::CRATE_HIR_ID, &c.attrs, DUMMY_SP);
+                        hir::OwnerNode::Crate(module)
+                    })
+                }
+                AstOwner::Item(item) => {
+                    self.with_lctx(item.id, |lctx| hir::OwnerNode::Item(lctx.lower_item(item)))
+                }
+                AstOwner::AssocItem(item, ctxt) => {
+                    self.with_lctx(item.id, |lctx| lctx.lower_assoc_item(item, ctxt))
+                }
+                AstOwner::ForeignItem(item) => self.with_lctx(item.id, |lctx| {
+                    hir::OwnerNode::ForeignItem(lctx.lower_foreign_item(item))
+                }),
             }
         }
-
-        self.owners[def_id]
-    }
-
-    #[instrument(level = "debug", skip(self, c))]
-    fn lower_crate(&mut self, c: &Crate) {
-        debug_assert_eq!(self.resolver.node_id_to_def_id[&CRATE_NODE_ID], CRATE_DEF_ID);
-        self.with_lctx(CRATE_NODE_ID, |lctx| {
-            let module = lctx.lower_mod(&c.items, &c.spans);
-            // FIXME(jdonszelman): is dummy span ever a problem here?
-            lctx.lower_attrs(hir::CRATE_HIR_ID, &c.attrs, DUMMY_SP);
-            hir::OwnerNode::Crate(module)
-        })
-    }
-
-    #[instrument(level = "debug", skip(self))]
-    fn lower_item(&mut self, item: &Item) {
-        self.with_lctx(item.id, |lctx| hir::OwnerNode::Item(lctx.lower_item(item)))
-    }
-
-    fn lower_assoc_item(&mut self, item: &AssocItem, ctxt: AssocCtxt) {
-        self.with_lctx(item.id, |lctx| lctx.lower_assoc_item(item, ctxt))
-    }
-
-    fn lower_foreign_item(&mut self, item: &ForeignItem) {
-        self.with_lctx(item.id, |lctx| hir::OwnerNode::ForeignItem(lctx.lower_foreign_item(item)))
     }
 }
 
@@ -192,7 +181,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 let (ty, body_id) =
                     self.lower_const_item(t, span, e.as_deref(), ImplTraitPosition::StaticTy);
                 self.lower_define_opaque(hir_id, define_opaque);
-                hir::ItemKind::Static(ident, ty, *m, body_id)
+                hir::ItemKind::Static(*m, ident, ty, body_id)
             }
             ItemKind::Const(box ast::ConstItem {
                 ident,
@@ -212,7 +201,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     },
                 );
                 self.lower_define_opaque(hir_id, &define_opaque);
-                hir::ItemKind::Const(ident, ty, generics, body_id)
+                hir::ItemKind::Const(ident, generics, ty, body_id)
             }
             ItemKind::Fn(box Fn {
                 sig: FnSig { decl, header, span: fn_sig_span },
@@ -316,9 +305,9 @@ impl<'hir> LoweringContext<'_, 'hir> {
                         ),
                     },
                 );
-                hir::ItemKind::TyAlias(ident, ty, generics)
+                hir::ItemKind::TyAlias(ident, generics, ty)
             }
-            ItemKind::Enum(ident, enum_definition, generics) => {
+            ItemKind::Enum(ident, generics, enum_definition) => {
                 let ident = self.lower_ident(*ident);
                 let (generics, variants) = self.lower_generics(
                     generics,
@@ -330,9 +319,9 @@ impl<'hir> LoweringContext<'_, 'hir> {
                         )
                     },
                 );
-                hir::ItemKind::Enum(ident, hir::EnumDef { variants }, generics)
+                hir::ItemKind::Enum(ident, generics, hir::EnumDef { variants })
             }
-            ItemKind::Struct(ident, struct_def, generics) => {
+            ItemKind::Struct(ident, generics, struct_def) => {
                 let ident = self.lower_ident(*ident);
                 let (generics, struct_def) = self.lower_generics(
                     generics,
@@ -340,9 +329,9 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     ImplTraitContext::Disallowed(ImplTraitPosition::Generic),
                     |this| this.lower_variant_data(hir_id, struct_def),
                 );
-                hir::ItemKind::Struct(ident, struct_def, generics)
+                hir::ItemKind::Struct(ident, generics, struct_def)
             }
-            ItemKind::Union(ident, vdata, generics) => {
+            ItemKind::Union(ident, generics, vdata) => {
                 let ident = self.lower_ident(*ident);
                 let (generics, vdata) = self.lower_generics(
                     generics,
@@ -350,7 +339,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     ImplTraitContext::Disallowed(ImplTraitPosition::Generic),
                     |this| this.lower_variant_data(hir_id, vdata),
                 );
-                hir::ItemKind::Union(ident, vdata, generics)
+                hir::ItemKind::Union(ident, generics, vdata)
             }
             ItemKind::Impl(box Impl {
                 safety,
@@ -479,8 +468,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
             ItemKind::Delegation(box delegation) => {
                 let delegation_results = self.lower_delegation(delegation, id, false);
                 hir::ItemKind::Fn {
-                    ident: delegation_results.ident,
                     sig: delegation_results.sig,
+                    ident: delegation_results.ident,
                     generics: delegation_results.generics,
                     body: delegation_results.body_id,
                     has_body: true,
@@ -539,7 +528,22 @@ impl<'hir> LoweringContext<'_, 'hir> {
             }
             UseTreeKind::Glob => {
                 let res = self.expect_full_res(id);
-                let res = smallvec![self.lower_res(res)];
+                let res = self.lower_res(res);
+                // Put the result in the appropriate namespace.
+                let res = match res {
+                    Res::Def(DefKind::Mod | DefKind::Trait, _) => {
+                        PerNS { type_ns: Some(res), value_ns: None, macro_ns: None }
+                    }
+                    Res::Def(DefKind::Enum, _) => {
+                        PerNS { type_ns: None, value_ns: Some(res), macro_ns: None }
+                    }
+                    Res::Err => {
+                        // Propagate the error to all namespaces, just to be sure.
+                        let err = Some(Res::Err);
+                        PerNS { type_ns: err, value_ns: err, macro_ns: err }
+                    }
+                    _ => span_bug!(path.span, "bad glob res {:?}", res),
+                };
                 let path = Path { segments, span: path.span, tokens: None };
                 let path = self.lower_use_path(res, &path, ParamMode::Explicit);
                 hir::ItemKind::Use(path, hir::UseKind::Glob)
@@ -613,7 +617,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 } else {
                     // For non-empty lists we can just drop all the data, the prefix is already
                     // present in HIR as a part of nested imports.
-                    self.arena.alloc(hir::UsePath { res: smallvec![], segments: &[], span })
+                    self.arena.alloc(hir::UsePath { res: PerNS::default(), segments: &[], span })
                 };
                 hir::ItemKind::Use(path, hir::UseKind::ListStem)
             }
diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs
index 19095f2e01e..b99df8bd7e5 100644
--- a/compiler/rustc_ast_lowering/src/lib.rs
+++ b/compiler/rustc_ast_lowering/src/lib.rs
@@ -33,9 +33,7 @@
 // tidy-alphabetical-start
 #![allow(internal_features)]
 #![doc(rust_logo)]
-#![feature(assert_matches)]
 #![feature(box_patterns)]
-#![feature(exact_size_is_empty)]
 #![feature(if_let_guard)]
 #![feature(rustdoc_internals)]
 // tidy-alphabetical-end
@@ -64,7 +62,7 @@ use rustc_middle::ty::{ResolverAstLowering, TyCtxt};
 use rustc_session::parse::{add_feature_diagnostics, feature_err};
 use rustc_span::symbol::{Ident, Symbol, kw, sym};
 use rustc_span::{DUMMY_SP, DesugaringKind, Span};
-use smallvec::{SmallVec, smallvec};
+use smallvec::SmallVec;
 use thin_vec::ThinVec;
 use tracing::{debug, instrument, trace};
 
@@ -444,14 +442,14 @@ pub fn lower_to_hir(tcx: TyCtxt<'_>, (): ()) -> hir::Crate<'_> {
         tcx.definitions_untracked().def_index_count(),
     );
 
+    let mut lowerer = item::ItemLowerer {
+        tcx,
+        resolver: &mut resolver,
+        ast_index: &ast_index,
+        owners: &mut owners,
+    };
     for def_id in ast_index.indices() {
-        item::ItemLowerer {
-            tcx,
-            resolver: &mut resolver,
-            ast_index: &ast_index,
-            owners: &mut owners,
-        }
-        .lower_node(def_id);
+        lowerer.lower_node(def_id);
     }
 
     drop(ast_index);
@@ -705,14 +703,16 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         self.resolver.get_partial_res(id).map_or(Res::Err, |pr| pr.expect_full_res())
     }
 
-    fn lower_import_res(&mut self, id: NodeId, span: Span) -> SmallVec<[Res; 3]> {
-        let res = self.resolver.get_import_res(id).present_items();
-        let res: SmallVec<_> = res.map(|res| self.lower_res(res)).collect();
-        if res.is_empty() {
+    fn lower_import_res(&mut self, id: NodeId, span: Span) -> PerNS<Option<Res>> {
+        let per_ns = self.resolver.get_import_res(id);
+        let per_ns = per_ns.map(|res| res.map(|res| self.lower_res(res)));
+        if per_ns.is_empty() {
+            // Propagate the error to all namespaces, just to be sure.
             self.dcx().span_delayed_bug(span, "no resolution for an import");
-            return smallvec![Res::Err];
+            let err = Some(Res::Err);
+            return PerNS { type_ns: err, value_ns: err, macro_ns: err };
         }
-        res
+        per_ns
     }
 
     fn make_lang_item_qpath(
@@ -730,7 +730,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         span: Span,
         args: Option<&'hir hir::GenericArgs<'hir>>,
     ) -> &'hir hir::Path<'hir> {
-        let def_id = self.tcx.require_lang_item(lang_item, Some(span));
+        let def_id = self.tcx.require_lang_item(lang_item, span);
         let def_kind = self.tcx.def_kind(def_id);
         let res = Res::Def(def_kind, def_id);
         self.arena.alloc(hir::Path {
@@ -1404,7 +1404,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 };
                 let span = self.tcx.sess.source_map().start_point(t.span).shrink_to_hi();
                 let region = Lifetime { ident: Ident::new(kw::UnderscoreLifetime, span), id };
-                (region, LifetimeSyntax::Hidden)
+                (region, LifetimeSyntax::Implicit)
             }
         };
         self.lower_lifetime(&region, LifetimeSource::Reference, syntax)
@@ -1788,7 +1788,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             id,
             Ident::new(kw::UnderscoreLifetime, span),
             LifetimeSource::Path { angle_brackets },
-            LifetimeSyntax::Hidden,
+            LifetimeSyntax::Implicit,
         )
     }
 
@@ -2420,7 +2420,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             Ident::new(kw::UnderscoreLifetime, self.lower_span(span)),
             hir::LifetimeKind::ImplicitObjectLifetimeDefault,
             LifetimeSource::Other,
-            LifetimeSyntax::Hidden,
+            LifetimeSyntax::Implicit,
         );
         debug!("elided_dyn_bound: r={:?}", r);
         self.arena.alloc(r)
diff --git a/compiler/rustc_ast_lowering/src/path.rs b/compiler/rustc_ast_lowering/src/path.rs
index 5cda64ce7b4..c80ef275c80 100644
--- a/compiler/rustc_ast_lowering/src/path.rs
+++ b/compiler/rustc_ast_lowering/src/path.rs
@@ -1,13 +1,13 @@
 use std::sync::Arc;
 
 use rustc_ast::{self as ast, *};
-use rustc_hir::def::{DefKind, PartialRes, Res};
+use rustc_hir::def::{DefKind, PartialRes, PerNS, Res};
 use rustc_hir::def_id::DefId;
 use rustc_hir::{self as hir, GenericArg};
 use rustc_middle::{span_bug, ty};
 use rustc_session::parse::add_feature_diagnostics;
 use rustc_span::{BytePos, DUMMY_SP, DesugaringKind, Ident, Span, Symbol, sym};
-use smallvec::{SmallVec, smallvec};
+use smallvec::smallvec;
 use tracing::{debug, instrument};
 
 use super::errors::{
@@ -226,11 +226,11 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
     pub(crate) fn lower_use_path(
         &mut self,
-        res: SmallVec<[Res; 3]>,
+        res: PerNS<Option<Res>>,
         p: &Path,
         param_mode: ParamMode,
     ) -> &'hir hir::UsePath<'hir> {
-        assert!((1..=3).contains(&res.len()));
+        assert!(!res.is_empty());
         self.arena.alloc(hir::UsePath {
             res,
             segments: self.arena.alloc_from_iter(p.segments.iter().map(|segment| {
diff --git a/compiler/rustc_ast_passes/src/ast_validation.rs b/compiler/rustc_ast_passes/src/ast_validation.rs
index cbf4f2f5eb2..d6fe04d2994 100644
--- a/compiler/rustc_ast_passes/src/ast_validation.rs
+++ b/compiler/rustc_ast_passes/src/ast_validation.rs
@@ -1010,7 +1010,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 });
                 self.extern_mod_span = old_item;
             }
-            ItemKind::Enum(_, def, _) => {
+            ItemKind::Enum(_, _, def) => {
                 for variant in &def.variants {
                     self.visibility_not_permitted(
                         &variant.vis,
@@ -1061,7 +1061,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 }
                 visit::walk_item(self, item)
             }
-            ItemKind::Struct(ident, vdata, generics) => match vdata {
+            ItemKind::Struct(ident, generics, vdata) => match vdata {
                 VariantData::Struct { fields, .. } => {
                     self.visit_attrs_vis_ident(&item.attrs, &item.vis, ident);
                     self.visit_generics(generics);
@@ -1070,7 +1070,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 }
                 _ => visit::walk_item(self, item),
             },
-            ItemKind::Union(ident, vdata, generics) => {
+            ItemKind::Union(ident, generics, vdata) => {
                 if vdata.fields().is_empty() {
                     self.dcx().emit_err(errors::FieldlessUnion { span: item.span });
                 }
diff --git a/compiler/rustc_ast_passes/src/feature_gate.rs b/compiler/rustc_ast_passes/src/feature_gate.rs
index 915613a3913..3682d25d341 100644
--- a/compiler/rustc_ast_passes/src/feature_gate.rs
+++ b/compiler/rustc_ast_passes/src/feature_gate.rs
@@ -477,11 +477,12 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session, features: &Features) {
         for span in spans {
             if (!visitor.features.coroutines() && !span.allows_unstable(sym::coroutines))
                 && (!visitor.features.gen_blocks() && !span.allows_unstable(sym::gen_blocks))
+                && (!visitor.features.yield_expr() && !span.allows_unstable(sym::yield_expr))
             {
                 #[allow(rustc::untranslatable_diagnostic)]
-                // Don't know which of the two features to include in the
-                // error message, so I am arbitrarily picking one.
-                feature_err(&visitor.sess, sym::coroutines, *span, "yield syntax is experimental")
+                // Emit yield_expr as the error, since that will be sufficient. You can think of it
+                // as coroutines and gen_blocks imply yield_expr.
+                feature_err(&visitor.sess, sym::yield_expr, *span, "yield syntax is experimental")
                     .emit();
             }
         }
diff --git a/compiler/rustc_ast_pretty/src/pprust/state/item.rs b/compiler/rustc_ast_pretty/src/pprust/state/item.rs
index 70cf2f2a459..3638eb31c61 100644
--- a/compiler/rustc_ast_pretty/src/pprust/state/item.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state/item.rs
@@ -298,14 +298,14 @@ impl<'a> State<'a> {
                     *defaultness,
                 );
             }
-            ast::ItemKind::Enum(ident, enum_definition, params) => {
-                self.print_enum_def(enum_definition, params, *ident, item.span, &item.vis);
+            ast::ItemKind::Enum(ident, generics, enum_definition) => {
+                self.print_enum_def(enum_definition, generics, *ident, item.span, &item.vis);
             }
-            ast::ItemKind::Struct(ident, struct_def, generics) => {
+            ast::ItemKind::Struct(ident, generics, struct_def) => {
                 let (cb, ib) = self.head(visibility_qualified(&item.vis, "struct"));
                 self.print_struct(struct_def, generics, *ident, item.span, true, cb, ib);
             }
-            ast::ItemKind::Union(ident, struct_def, generics) => {
+            ast::ItemKind::Union(ident, generics, struct_def) => {
                 let (cb, ib) = self.head(visibility_qualified(&item.vis, "union"));
                 self.print_struct(struct_def, generics, *ident, item.span, true, cb, ib);
             }
diff --git a/compiler/rustc_attr_data_structures/src/version.rs b/compiler/rustc_attr_data_structures/src/version.rs
index 69b0e041d81..030e9520940 100644
--- a/compiler/rustc_attr_data_structures/src/version.rs
+++ b/compiler/rustc_attr_data_structures/src/version.rs
@@ -1,4 +1,5 @@
 use std::fmt::{self, Display};
+use std::sync::OnceLock;
 
 use rustc_macros::{
     Decodable, Encodable, HashStable_Generic, PrintAttribute, current_rustc_version,
@@ -16,8 +17,29 @@ pub struct RustcVersion {
 
 impl RustcVersion {
     pub const CURRENT: Self = current_rustc_version!();
+    pub fn current_overridable() -> Self {
+        *CURRENT_OVERRIDABLE.get_or_init(|| {
+            if let Ok(override_var) = std::env::var("RUSTC_OVERRIDE_VERSION_STRING")
+                && let Some(override_) = Self::parse_str(&override_var)
+            {
+                override_
+            } else {
+                Self::CURRENT
+            }
+        })
+    }
+    fn parse_str(value: &str) -> Option<Self> {
+        // Ignore any suffixes such as "-dev" or "-nightly".
+        let mut components = value.split('-').next().unwrap().splitn(3, '.');
+        let major = components.next()?.parse().ok()?;
+        let minor = components.next()?.parse().ok()?;
+        let patch = components.next().unwrap_or("0").parse().ok()?;
+        Some(RustcVersion { major, minor, patch })
+    }
 }
 
+static CURRENT_OVERRIDABLE: OnceLock<RustcVersion> = OnceLock::new();
+
 impl Display for RustcVersion {
     fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(formatter, "{}.{}.{}", self.major, self.minor, self.patch)
diff --git a/compiler/rustc_attr_parsing/src/attributes/allow_unstable.rs b/compiler/rustc_attr_parsing/src/attributes/allow_unstable.rs
index c1d95d07f4c..d0465546b73 100644
--- a/compiler/rustc_attr_parsing/src/attributes/allow_unstable.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/allow_unstable.rs
@@ -10,7 +10,7 @@ use crate::session_diagnostics;
 
 pub(crate) struct AllowInternalUnstableParser;
 impl CombineAttributeParser for AllowInternalUnstableParser {
-    const PATH: &'static [rustc_span::Symbol] = &[sym::allow_internal_unstable];
+    const PATH: &'static [Symbol] = &[sym::allow_internal_unstable];
     type Item = (Symbol, Span);
     const CONVERT: ConvertFn<Self::Item> = AttributeKind::AllowInternalUnstable;
 
@@ -24,7 +24,7 @@ impl CombineAttributeParser for AllowInternalUnstableParser {
 
 pub(crate) struct AllowConstFnUnstableParser;
 impl CombineAttributeParser for AllowConstFnUnstableParser {
-    const PATH: &'static [rustc_span::Symbol] = &[sym::rustc_allow_const_fn_unstable];
+    const PATH: &'static [Symbol] = &[sym::rustc_allow_const_fn_unstable];
     type Item = Symbol;
     const CONVERT: ConvertFn<Self::Item> = AttributeKind::AllowConstFnUnstable;
 
@@ -53,7 +53,7 @@ fn parse_unstable<'a>(
 
     for param in list.mixed() {
         let param_span = param.span();
-        if let Some(ident) = param.meta_item().and_then(|i| i.path_without_args().word()) {
+        if let Some(ident) = param.meta_item().and_then(|i| i.path().word()) {
             res.push(ident.name);
         } else {
             cx.emit_err(session_diagnostics::ExpectsFeatures {
diff --git a/compiler/rustc_attr_parsing/src/attributes/cfg.rs b/compiler/rustc_attr_parsing/src/attributes/cfg.rs
index 7cb1fede174..f4d23012af7 100644
--- a/compiler/rustc_attr_parsing/src/attributes/cfg.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/cfg.rs
@@ -4,14 +4,29 @@ use rustc_attr_data_structures::RustcVersion;
 use rustc_feature::{Features, GatedCfg, find_gated_cfg};
 use rustc_session::Session;
 use rustc_session::config::ExpectedValues;
-use rustc_session::lint::BuiltinLintDiag;
 use rustc_session::lint::builtin::UNEXPECTED_CFGS;
+use rustc_session::lint::{BuiltinLintDiag, Lint};
 use rustc_session::parse::feature_err;
 use rustc_span::{Span, Symbol, sym};
 
 use crate::session_diagnostics::{self, UnsupportedLiteralReason};
 use crate::{fluent_generated, parse_version};
 
+/// Emitter of a builtin lint from `cfg_matches`.
+///
+/// Used to support emiting a lint (currently on check-cfg), either:
+///  - as an early buffered lint (in `rustc`)
+///  - or has a "normal" lint from HIR (in `rustdoc`)
+pub trait CfgMatchesLintEmitter {
+    fn emit_span_lint(&self, sess: &Session, lint: &'static Lint, sp: Span, diag: BuiltinLintDiag);
+}
+
+impl CfgMatchesLintEmitter for NodeId {
+    fn emit_span_lint(&self, sess: &Session, lint: &'static Lint, sp: Span, diag: BuiltinLintDiag) {
+        sess.psess.buffer_lint(lint, sp, *self, diag);
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct Condition {
     pub name: Symbol,
@@ -25,17 +40,17 @@ pub struct Condition {
 pub fn cfg_matches(
     cfg: &MetaItemInner,
     sess: &Session,
-    lint_node_id: NodeId,
+    lint_emitter: impl CfgMatchesLintEmitter,
     features: Option<&Features>,
 ) -> bool {
     eval_condition(cfg, sess, features, &mut |cfg| {
         try_gate_cfg(cfg.name, cfg.span, sess, features);
         match sess.psess.check_config.expecteds.get(&cfg.name) {
             Some(ExpectedValues::Some(values)) if !values.contains(&cfg.value) => {
-                sess.psess.buffer_lint(
+                lint_emitter.emit_span_lint(
+                    sess,
                     UNEXPECTED_CFGS,
                     cfg.span,
-                    lint_node_id,
                     BuiltinLintDiag::UnexpectedCfgValue(
                         (cfg.name, cfg.name_span),
                         cfg.value.map(|v| (v, cfg.value_span.unwrap())),
@@ -43,10 +58,10 @@ pub fn cfg_matches(
                 );
             }
             None if sess.psess.check_config.exhaustive_names => {
-                sess.psess.buffer_lint(
+                lint_emitter.emit_span_lint(
+                    sess,
                     UNEXPECTED_CFGS,
                     cfg.span,
-                    lint_node_id,
                     BuiltinLintDiag::UnexpectedCfgName(
                         (cfg.name, cfg.name_span),
                         cfg.value.map(|v| (v, cfg.value_span.unwrap())),
@@ -129,9 +144,9 @@ pub fn eval_condition(
 
             // See https://github.com/rust-lang/rust/issues/64796#issuecomment-640851454 for details
             if sess.psess.assume_incomplete_release {
-                RustcVersion::CURRENT > min_version
+                RustcVersion::current_overridable() > min_version
             } else {
-                RustcVersion::CURRENT >= min_version
+                RustcVersion::current_overridable() >= min_version
             }
         }
         MetaItemKind::List(mis) => {
diff --git a/compiler/rustc_attr_parsing/src/attributes/deprecation.rs b/compiler/rustc_attr_parsing/src/attributes/deprecation.rs
index fb3d5f57d4f..006c1fe3b9c 100644
--- a/compiler/rustc_attr_parsing/src/attributes/deprecation.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/deprecation.rs
@@ -42,9 +42,9 @@ fn get(
 }
 
 impl SingleAttributeParser for DeprecationParser {
-    const PATH: &'static [rustc_span::Symbol] = &[sym::deprecated];
+    const PATH: &'static [Symbol] = &[sym::deprecated];
 
-    fn on_duplicate(cx: &AcceptContext<'_>, first_span: rustc_span::Span) {
+    fn on_duplicate(cx: &AcceptContext<'_>, first_span: Span) {
         // FIXME(jdonszelmann): merge with errors from check_attrs.rs
         cx.emit_err(session_diagnostics::UnusedMultiple {
             this: cx.attr_span,
@@ -79,7 +79,7 @@ impl SingleAttributeParser for DeprecationParser {
                     return None;
                 };
 
-                let ident_name = param.path_without_args().word_sym();
+                let ident_name = param.path().word_sym();
 
                 match ident_name {
                     Some(name @ sym::since) => {
@@ -102,7 +102,7 @@ impl SingleAttributeParser for DeprecationParser {
                     _ => {
                         cx.emit_err(session_diagnostics::UnknownMetaItem {
                             span: param_span,
-                            item: param.path_without_args().to_string(),
+                            item: param.path().to_string(),
                             expected: if features.deprecated_suggestion() {
                                 &["since", "note", "suggestion"]
                             } else {
diff --git a/compiler/rustc_attr_parsing/src/attributes/mod.rs b/compiler/rustc_attr_parsing/src/attributes/mod.rs
index 6ecd6b4d7db..bf18e10e19f 100644
--- a/compiler/rustc_attr_parsing/src/attributes/mod.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/mod.rs
@@ -17,7 +17,7 @@
 use std::marker::PhantomData;
 
 use rustc_attr_data_structures::AttributeKind;
-use rustc_span::Span;
+use rustc_span::{Span, Symbol};
 use thin_vec::ThinVec;
 
 use crate::context::{AcceptContext, FinalizeContext};
@@ -33,7 +33,7 @@ pub(crate) mod transparency;
 pub(crate) mod util;
 
 type AcceptFn<T> = fn(&mut T, &AcceptContext<'_>, &ArgParser<'_>);
-type AcceptMapping<T> = &'static [(&'static [rustc_span::Symbol], AcceptFn<T>)];
+type AcceptMapping<T> = &'static [(&'static [Symbol], AcceptFn<T>)];
 
 /// An [`AttributeParser`] is a type which searches for syntactic attributes.
 ///
@@ -72,9 +72,9 @@ pub(crate) trait AttributeParser: Default + 'static {
 /// [`SingleAttributeParser`] can only convert attributes one-to-one, and cannot combine multiple
 /// attributes together like is necessary for `#[stable()]` and `#[unstable()]` for example.
 pub(crate) trait SingleAttributeParser: 'static {
-    const PATH: &'static [rustc_span::Symbol];
+    const PATH: &'static [Symbol];
 
-    /// Caled when a duplicate attribute is found.
+    /// Called when a duplicate attribute is found.
     ///
     /// `first_span` is the span of the first occurrence of this attribute.
     // FIXME(jdonszelmann): default error
@@ -119,7 +119,7 @@ type ConvertFn<E> = fn(ThinVec<E>) -> AttributeKind;
 /// [`CombineAttributeParser`] can only convert a single kind of attribute, and cannot combine multiple
 /// attributes together like is necessary for `#[stable()]` and `#[unstable()]` for example.
 pub(crate) trait CombineAttributeParser: 'static {
-    const PATH: &'static [rustc_span::Symbol];
+    const PATH: &'static [Symbol];
 
     type Item;
     const CONVERT: ConvertFn<Self::Item>;
diff --git a/compiler/rustc_attr_parsing/src/attributes/repr.rs b/compiler/rustc_attr_parsing/src/attributes/repr.rs
index 43dfb85a7c4..69316541e19 100644
--- a/compiler/rustc_attr_parsing/src/attributes/repr.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/repr.rs
@@ -1,7 +1,7 @@
 use rustc_abi::Align;
 use rustc_ast::{IntTy, LitIntType, LitKind, UintTy};
 use rustc_attr_data_structures::{AttributeKind, IntType, ReprAttr};
-use rustc_span::{Span, Symbol, sym};
+use rustc_span::{DUMMY_SP, Span, Symbol, sym};
 
 use super::{CombineAttributeParser, ConvertFn};
 use crate::context::AcceptContext;
@@ -21,7 +21,7 @@ pub(crate) struct ReprParser;
 
 impl CombineAttributeParser for ReprParser {
     type Item = (ReprAttr, Span);
-    const PATH: &'static [rustc_span::Symbol] = &[sym::repr];
+    const PATH: &'static [Symbol] = &[sym::repr];
     const CONVERT: ConvertFn<Self::Item> = AttributeKind::Repr;
 
     fn extend<'a>(
@@ -96,10 +96,10 @@ fn parse_repr(cx: &AcceptContext<'_>, param: &MetaItemParser<'_>) -> Option<Repr
 
     // FIXME(jdonszelmann): invert the parsing here to match on the word first and then the
     // structure.
-    let (name, ident_span) = if let Some(ident) = param.path_without_args().word() {
+    let (name, ident_span) = if let Some(ident) = param.path().word() {
         (Some(ident.name), ident.span)
     } else {
-        (None, rustc_span::DUMMY_SP)
+        (None, DUMMY_SP)
     };
 
     let args = param.args();
diff --git a/compiler/rustc_attr_parsing/src/attributes/stability.rs b/compiler/rustc_attr_parsing/src/attributes/stability.rs
index cd1f21d92e7..ce69a54513d 100644
--- a/compiler/rustc_attr_parsing/src/attributes/stability.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/stability.rs
@@ -117,7 +117,7 @@ impl AttributeParser for BodyStabilityParser {
 pub(crate) struct ConstStabilityIndirectParser;
 // FIXME(jdonszelmann): single word attribute group when we have these
 impl SingleAttributeParser for ConstStabilityIndirectParser {
-    const PATH: &'static [rustc_span::Symbol] = &[sym::rustc_const_stable_indirect];
+    const PATH: &'static [Symbol] = &[sym::rustc_const_stable_indirect];
 
     // ignore
     fn on_duplicate(_cx: &AcceptContext<'_>, _first_span: Span) {}
@@ -204,7 +204,7 @@ fn insert_value_into_option_or_error(
     if item.is_some() {
         cx.emit_err(session_diagnostics::MultipleItem {
             span: param.span(),
-            item: param.path_without_args().to_string(),
+            item: param.path().to_string(),
         });
         None
     } else if let Some(v) = param.args().name_value()
@@ -242,13 +242,13 @@ pub(crate) fn parse_stability(
             return None;
         };
 
-        match param.path_without_args().word_sym() {
+        match param.path().word_sym() {
             Some(sym::feature) => insert_value_into_option_or_error(cx, &param, &mut feature)?,
             Some(sym::since) => insert_value_into_option_or_error(cx, &param, &mut since)?,
             _ => {
                 cx.emit_err(session_diagnostics::UnknownMetaItem {
                     span: param_span,
-                    item: param.path_without_args().to_string(),
+                    item: param.path().to_string(),
                     expected: &["feature", "since"],
                 });
                 return None;
@@ -310,7 +310,7 @@ pub(crate) fn parse_unstability(
             return None;
         };
 
-        match param.path_without_args().word_sym() {
+        match param.path().word_sym() {
             Some(sym::feature) => insert_value_into_option_or_error(cx, &param, &mut feature)?,
             Some(sym::reason) => insert_value_into_option_or_error(cx, &param, &mut reason)?,
             Some(sym::issue) => {
@@ -349,7 +349,7 @@ pub(crate) fn parse_unstability(
             _ => {
                 cx.emit_err(session_diagnostics::UnknownMetaItem {
                     span: param.span(),
-                    item: param.path_without_args().to_string(),
+                    item: param.path().to_string(),
                     expected: &["feature", "reason", "issue", "soft", "implied_by"],
                 });
                 return None;
diff --git a/compiler/rustc_attr_parsing/src/attributes/transparency.rs b/compiler/rustc_attr_parsing/src/attributes/transparency.rs
index ce42b0507ed..d229fc09740 100644
--- a/compiler/rustc_attr_parsing/src/attributes/transparency.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/transparency.rs
@@ -1,6 +1,6 @@
 use rustc_attr_data_structures::AttributeKind;
 use rustc_span::hygiene::Transparency;
-use rustc_span::sym;
+use rustc_span::{Span, Symbol, sym};
 
 use super::{AcceptContext, SingleAttributeParser};
 use crate::parser::ArgParser;
@@ -11,9 +11,9 @@ pub(crate) struct TransparencyParser;
 #[allow(rustc::untranslatable_diagnostic)]
 #[allow(rustc::diagnostic_outside_of_impl)]
 impl SingleAttributeParser for TransparencyParser {
-    const PATH: &'static [rustc_span::Symbol] = &[sym::rustc_macro_transparency];
+    const PATH: &'static [Symbol] = &[sym::rustc_macro_transparency];
 
-    fn on_duplicate(cx: &crate::context::AcceptContext<'_>, first_span: rustc_span::Span) {
+    fn on_duplicate(cx: &crate::context::AcceptContext<'_>, first_span: Span) {
         cx.dcx().span_err(vec![first_span, cx.attr_span], "multiple macro transparency attributes");
     }
 
diff --git a/compiler/rustc_attr_parsing/src/attributes/util.rs b/compiler/rustc_attr_parsing/src/attributes/util.rs
index 05a9029c59a..503d2f1fae1 100644
--- a/compiler/rustc_attr_parsing/src/attributes/util.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/util.rs
@@ -26,3 +26,33 @@ pub fn is_builtin_attr(attr: &impl AttributeExt) -> bool {
 pub fn find_crate_name(attrs: &[impl AttributeExt]) -> Option<Symbol> {
     first_attr_value_str_by_name(attrs, sym::crate_name)
 }
+
+pub fn is_doc_alias_attrs_contain_symbol<'tcx, T: AttributeExt + 'tcx>(
+    attrs: impl Iterator<Item = &'tcx T>,
+    symbol: Symbol,
+) -> bool {
+    let doc_attrs = attrs.filter(|attr| attr.has_name(sym::doc));
+    for attr in doc_attrs {
+        let Some(values) = attr.meta_item_list() else {
+            continue;
+        };
+        let alias_values = values.iter().filter(|v| v.has_name(sym::alias));
+        for v in alias_values {
+            if let Some(nested) = v.meta_item_list() {
+                // #[doc(alias("foo", "bar"))]
+                let mut iter = nested.iter().filter_map(|item| item.lit()).map(|item| item.symbol);
+                if iter.any(|s| s == symbol) {
+                    return true;
+                }
+            } else if let Some(meta) = v.meta_item()
+                && let Some(lit) = meta.name_value_literal()
+            {
+                // #[doc(alias = "foo")]
+                if lit.symbol == symbol {
+                    return true;
+                }
+            }
+        }
+    }
+    false
+}
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index 55c3df003fe..c02760d830c 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -26,12 +26,16 @@ macro_rules! attribute_groups {
     (
         pub(crate) static $name: ident = [$($names: ty),* $(,)?];
     ) => {
-        pub(crate) static $name: LazyLock<(
-            BTreeMap<&'static [Symbol], Vec<Box<dyn Fn(&AcceptContext<'_>, &ArgParser<'_>) + Send + Sync>>>,
-            Vec<Box<dyn Send + Sync + Fn(&FinalizeContext<'_>) -> Option<AttributeKind>>>
-        )> = LazyLock::new(|| {
-            let mut accepts = BTreeMap::<_, Vec<Box<dyn Fn(&AcceptContext<'_>, &ArgParser<'_>) + Send + Sync>>>::new();
-            let mut finalizes = Vec::<Box<dyn Send + Sync + Fn(&FinalizeContext<'_>) -> Option<AttributeKind>>>::new();
+        type Accepts = BTreeMap<
+            &'static [Symbol],
+            Box<dyn Send + Sync + Fn(&AcceptContext<'_>, &ArgParser<'_>)>
+        >;
+        type Finalizes = Vec<
+            Box<dyn Send + Sync + Fn(&FinalizeContext<'_>) -> Option<AttributeKind>>
+        >;
+        pub(crate) static $name: LazyLock<(Accepts, Finalizes)> = LazyLock::new(|| {
+            let mut accepts = Accepts::new();
+            let mut finalizes = Finalizes::new();
             $(
                 {
                     thread_local! {
@@ -39,11 +43,12 @@ macro_rules! attribute_groups {
                     };
 
                     for (k, v) in <$names>::ATTRIBUTES {
-                        accepts.entry(*k).or_default().push(Box::new(|cx, args| {
+                        let old = accepts.insert(*k, Box::new(|cx, args| {
                             STATE_OBJECT.with_borrow_mut(|s| {
                                 v(s, cx, args)
                             })
                         }));
+                        assert!(old.is_none());
                     }
 
                     finalizes.push(Box::new(|cx| {
@@ -110,7 +115,8 @@ impl<'a> Deref for AcceptContext<'a> {
 
 /// Context given to every attribute parser during finalization.
 ///
-/// Gives [`AttributeParser`](crate::attributes::AttributeParser)s enough information to create errors, for example.
+/// Gives [`AttributeParser`](crate::attributes::AttributeParser)s enough information to create
+/// errors, for example.
 pub(crate) struct FinalizeContext<'a> {
     /// The parse context, gives access to the session and the
     /// diagnostics context.
@@ -141,10 +147,9 @@ pub struct AttributeParser<'sess> {
     sess: &'sess Session,
     features: Option<&'sess Features>,
 
-    /// *only* parse attributes with this symbol.
+    /// *Only* parse attributes with this symbol.
     ///
-    /// Used in cases where we want the lowering infrastructure for
-    /// parse just a single attribute.
+    /// Used in cases where we want the lowering infrastructure for parse just a single attribute.
     parse_only: Option<Symbol>,
 
     /// Can be used to instruct parsers to reduce the number of diagnostics it emits.
@@ -157,9 +162,9 @@ impl<'sess> AttributeParser<'sess> {
     /// One example where this is necessary, is to parse `feature` attributes themselves for
     /// example.
     ///
-    /// Try to use this as little as possible. Attributes *should* be lowered during `rustc_ast_lowering`.
-    /// Some attributes require access to features to parse, which would crash if you tried to do so
-    /// through [`parse_limited`](Self::parse_limited).
+    /// Try to use this as little as possible. Attributes *should* be lowered during
+    /// `rustc_ast_lowering`. Some attributes require access to features to parse, which would
+    /// crash if you tried to do so through [`parse_limited`](Self::parse_limited).
     ///
     /// To make sure use is limited, supply a `Symbol` you'd like to parse. Only attributes with
     /// that symbol are picked out of the list of instructions and parsed. Those are returned.
@@ -217,19 +222,18 @@ impl<'sess> AttributeParser<'sess> {
         let group_cx = FinalizeContext { cx: self, target_span };
 
         for attr in attrs {
-            // if we're only looking for a single attribute,
-            // skip all the ones we don't care about
+            // If we're only looking for a single attribute, skip all the ones we don't care about.
             if let Some(expected) = self.parse_only {
                 if !attr.has_name(expected) {
                     continue;
                 }
             }
 
-            // sometimes, for example for `#![doc = include_str!("readme.md")]`,
+            // Sometimes, for example for `#![doc = include_str!("readme.md")]`,
             // doc still contains a non-literal. You might say, when we're lowering attributes
             // that's expanded right? But no, sometimes, when parsing attributes on macros,
             // we already use the lowering logic and these are still there. So, when `omit_doc`
-            // is set we *also* want to ignore these
+            // is set we *also* want to ignore these.
             if omit_doc == OmitDoc::Skip && attr.has_name(sym::doc) {
                 continue;
             }
@@ -260,24 +264,21 @@ impl<'sess> AttributeParser<'sess> {
                 // }
                 ast::AttrKind::Normal(n) => {
                     let parser = MetaItemParser::from_attr(n, self.dcx());
-                    let (path, args) = parser.deconstruct();
+                    let path = parser.path();
+                    let args = parser.args();
                     let parts = path.segments().map(|i| i.name).collect::<Vec<_>>();
 
-                    if let Some(accepts) = ATTRIBUTE_MAPPING.0.get(parts.as_slice()) {
-                        for f in accepts {
-                            let cx = AcceptContext {
-                                group_cx: &group_cx,
-                                attr_span: lower_span(attr.span),
-                            };
+                    if let Some(accept) = ATTRIBUTE_MAPPING.0.get(parts.as_slice()) {
+                        let cx =
+                            AcceptContext { group_cx: &group_cx, attr_span: lower_span(attr.span) };
 
-                            f(&cx, &args)
-                        }
+                        accept(&cx, &args)
                     } else {
-                        // if we're here, we must be compiling a tool attribute... Or someone forgot to
-                        // parse their fancy new attribute. Let's warn them in any case. If you are that
-                        // person, and you really your attribute should remain unparsed, carefully read the
-                        // documentation in this module and if you still think so you can add an exception
-                        // to this assertion.
+                        // If we're here, we must be compiling a tool attribute... Or someone
+                        // forgot to parse their fancy new attribute. Let's warn them in any case.
+                        // If you are that person, and you really think your attribute should
+                        // remain unparsed, carefully read the documentation in this module and if
+                        // you still think so you can add an exception to this assertion.
 
                         // FIXME(jdonszelmann): convert other attributes, and check with this that
                         // we caught em all
diff --git a/compiler/rustc_attr_parsing/src/lib.rs b/compiler/rustc_attr_parsing/src/lib.rs
index da683ad58c1..15037e802ff 100644
--- a/compiler/rustc_attr_parsing/src/lib.rs
+++ b/compiler/rustc_attr_parsing/src/lib.rs
@@ -1,31 +1,38 @@
 //! Centralized logic for parsing and attributes.
 //!
-//! Part of a series of crates:
-//! - rustc_attr_data_structures: contains types that the parsers parse into
-//! - rustc_attr_parsing: this crate
-//! - (in the future): rustc_attr_validation
+//! ## Architecture
+//! This crate is part of a series of crates that handle attribute processing.
+//! - [rustc_attr_data_structures](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_attr_data_structures/index.html): Defines the data structures that store parsed attributes
+//! - [rustc_attr_parsing](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_attr_parsing/index.html): This crate, handles the parsing of attributes
+//! - (planned) rustc_attr_validation: Will handle attribute validation
 //!
-//! History: Check out [#131229](https://github.com/rust-lang/rust/issues/131229).
-//! There used to be only one definition of attributes in the compiler: `ast::Attribute`.
-//! These were then parsed or validated or both in places distributed all over the compiler.
-//! This was a mess...
+//! The separation between data structures and parsing follows the principle of separation of concerns.
+//! Data structures (`rustc_attr_data_structures`) define what attributes look like after parsing.
+//! This crate (`rustc_attr_parsing`) handles how to convert raw tokens into those structures.
+//! This split allows other parts of the compiler to use the data structures without needing
+//! the parsing logic, making the codebase more modular and maintainable.
 //!
-//! Attributes are markers on items.
-//! Many of them are actually attribute-like proc-macros, and are expanded to some other rust syntax.
-//! This could either be a user provided proc macro, or something compiler provided.
-//! `derive` is an example of one that the compiler provides.
-//! These are built-in, but they have a valid expansion to Rust tokens and are thus called "active".
-//! I personally like calling these *active* compiler-provided attributes, built-in *macros*,
-//! because they still expand, and this helps to differentiate them from built-in *attributes*.
-//! However, I'll be the first to admit that the naming here can be confusing.
+//! ## Background
+//! Previously, the compiler had a single attribute definition (`ast::Attribute`) with parsing and
+//! validation scattered throughout the codebase. This was reorganized for better maintainability
+//! (see [#131229](https://github.com/rust-lang/rust/issues/131229)).
 //!
-//! The alternative to active attributes, are inert attributes.
-//! These can occur in user code (proc-macro helper attributes).
-//! But what's important is, many built-in attributes are inert like this.
-//! There is nothing they expand to during the macro expansion process,
-//! sometimes because they literally cannot expand to something that is valid Rust.
-//! They are really just markers to guide the compilation process.
-//! An example is `#[inline(...)]` which changes how code for functions is generated.
+//! ## Types of Attributes
+//! In Rust, attributes are markers that can be attached to items. They come in two main categories.
+//!
+//! ### 1. Active Attributes
+//! These are attribute-like proc-macros that expand into other Rust code.
+//! They can be either user-defined or compiler-provided. Examples of compiler-provided active attributes:
+//!   - `#[derive(...)]`: Expands into trait implementations
+//!   - `#[cfg()]`: Expands based on configuration
+//!   - `#[cfg_attr()]`: Conditional attribute application
+//!
+//! ### 2. Inert Attributes
+//! These are pure markers that don't expand into other code. They guide the compilation process.
+//! They can be user-defined (in proc-macro helpers) or built-in. Examples of built-in inert attributes:
+//!   - `#[stable()]`: Marks stable API items
+//!   - `#[inline()]`: Suggests function inlining
+//!   - `#[repr()]`: Controls type representation
 //!
 //! ```text
 //!                      Active                 Inert
@@ -33,27 +40,21 @@
 //!              │     (mostly in)      │    these are parsed  │
 //!              │ rustc_builtin_macros │        here!         │
 //!              │                      │                      │
-//!              │                      │                      │
 //!              │    #[derive(...)]    │    #[stable()]       │
 //!     Built-in │    #[cfg()]          │    #[inline()]       │
 //!              │    #[cfg_attr()]     │    #[repr()]         │
 //!              │                      │                      │
-//!              │                      │                      │
-//!              │                      │                      │
 //!              ├──────────────────────┼──────────────────────┤
 //!              │                      │                      │
-//!              │                      │                      │
 //!              │                      │       `b` in         │
 //!              │                      │ #[proc_macro_derive( │
 //! User created │ #[proc_macro_attr()] │    a,                │
 //!              │                      │    attributes(b)     │
 //!              │                      │ ]                    │
-//!              │                      │                      │
-//!              │                      │                      │
-//!              │                      │                      │
 //!              └──────────────────────┴──────────────────────┘
 //! ```
 //!
+//! ## How This Crate Works
 //! In this crate, syntactical attributes (sequences of tokens that look like
 //! `#[something(something else)]`) are parsed into more semantic attributes, markers on items.
 //! Multiple syntactic attributes might influence a single semantic attribute. For example,
@@ -63,18 +64,17 @@
 //! and `#[unstable()]` syntactic attributes, and at the end produce a single
 //! [`AttributeKind::Stability`](rustc_attr_data_structures::AttributeKind::Stability).
 //!
-//! As a rule of thumb, when a syntactical attribute can be applied more than once, they should be
-//! combined into a single semantic attribute. For example:
+//! When multiple instances of the same attribute are allowed, they're combined into a single
+//! semantic attribute. For example:
 //!
-//! ```
+//! ```rust
 //! #[repr(C)]
 //! #[repr(packed)]
 //! struct Meow {}
 //! ```
 //!
-//! should result in a single `AttributeKind::Repr` containing a list of repr annotations, in this
-//! case `C` and `packed`. This is equivalent to writing `#[repr(C, packed)]` in a single
-//! syntactical annotation.
+//! This is equivalent to `#[repr(C, packed)]` and results in a single `AttributeKind::Repr`
+//! containing both `C` and `packed` annotations.
 
 // tidy-alphabetical-start
 #![allow(internal_features)]
@@ -90,7 +90,9 @@ pub mod parser;
 mod session_diagnostics;
 
 pub use attributes::cfg::*;
-pub use attributes::util::{find_crate_name, is_builtin_attr, parse_version};
+pub use attributes::util::{
+    find_crate_name, is_builtin_attr, is_doc_alias_attrs_contain_symbol, parse_version,
+};
 pub use context::{AttributeParser, OmitDoc};
 
 rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_attr_parsing/src/parser.rs b/compiler/rustc_attr_parsing/src/parser.rs
index 077d953cfa3..e10e3b511db 100644
--- a/compiler/rustc_attr_parsing/src/parser.rs
+++ b/compiler/rustc_attr_parsing/src/parser.rs
@@ -12,8 +12,7 @@ use rustc_ast::{AttrArgs, DelimArgs, Expr, ExprKind, LitKind, MetaItemLit, Norma
 use rustc_ast_pretty::pprust;
 use rustc_errors::DiagCtxtHandle;
 use rustc_hir::{self as hir, AttrPath};
-use rustc_span::symbol::{Ident, kw, sym};
-use rustc_span::{ErrorGuaranteed, Span, Symbol};
+use rustc_span::{ErrorGuaranteed, Ident, Span, Symbol, kw, sym};
 
 pub struct SegmentIterator<'a> {
     offset: usize,
@@ -253,35 +252,18 @@ impl<'a> MetaItemParser<'a> {
         }
     }
 
-    /// Gets just the path, without the args.
-    pub fn path_without_args(&self) -> PathParser<'a> {
-        self.path.clone()
-    }
-
-    /// Gets just the args parser, without caring about the path.
-    pub fn args(&self) -> &ArgParser<'a> {
-        &self.args
-    }
-
-    pub fn deconstruct(&self) -> (PathParser<'a>, &ArgParser<'a>) {
-        (self.path_without_args(), self.args())
-    }
-
-    /// Asserts that this MetaItem starts with a path. Some examples:
+    /// Gets just the path, without the args. Some examples:
     ///
     /// - `#[rustfmt::skip]`: `rustfmt::skip` is a path
     /// - `#[allow(clippy::complexity)]`: `clippy::complexity` is a path
     /// - `#[inline]`: `inline` is a single segment path
-    pub fn path(&self) -> (PathParser<'a>, &ArgParser<'a>) {
-        self.deconstruct()
+    pub fn path(&self) -> &PathParser<'a> {
+        &self.path
     }
 
-    /// Asserts that this MetaItem starts with a word, or single segment path.
-    /// Doesn't return the args parser.
-    ///
-    /// For examples. see [`Self::word`]
-    pub fn word_without_args(&self) -> Option<Ident> {
-        Some(self.word()?.0)
+    /// Gets just the args parser, without caring about the path.
+    pub fn args(&self) -> &ArgParser<'a> {
+        &self.args
     }
 
     /// Asserts that this MetaItem starts with a word, or single segment path.
@@ -290,23 +272,8 @@ impl<'a> MetaItemParser<'a> {
     /// - `#[inline]`: `inline` is a word
     /// - `#[rustfmt::skip]`: `rustfmt::skip` is a path,
     ///   and not a word and should instead be parsed using [`path`](Self::path)
-    pub fn word(&self) -> Option<(Ident, &ArgParser<'a>)> {
-        let (path, args) = self.deconstruct();
-        Some((path.word()?, args))
-    }
-
-    /// Asserts that this MetaItem starts with some specific word.
-    ///
-    /// See [`word`](Self::word) for examples of what a word is.
     pub fn word_is(&self, sym: Symbol) -> Option<&ArgParser<'a>> {
-        self.path_without_args().word_is(sym).then(|| self.args())
-    }
-
-    /// Asserts that this MetaItem starts with some specific path.
-    ///
-    /// See [`word`](Self::path) for examples of what a word is.
-    pub fn path_is(&self, segments: &[Symbol]) -> Option<&ArgParser<'a>> {
-        self.path_without_args().segments_is(segments).then(|| self.args())
+        self.path().word_is(sym).then(|| self.args())
     }
 }
 
@@ -549,7 +516,7 @@ impl<'a> MetaItemListParser<'a> {
     }
 
     /// Lets you pick and choose as what you want to parse each element in the list
-    pub fn mixed<'s>(&'s self) -> impl Iterator<Item = &'s MetaItemOrLitParser<'a>> + 's {
+    pub fn mixed(&self) -> impl Iterator<Item = &MetaItemOrLitParser<'a>> {
         self.sub_parsers.iter()
     }
 
@@ -561,20 +528,6 @@ impl<'a> MetaItemListParser<'a> {
         self.len() == 0
     }
 
-    /// Asserts that every item in the list is another list starting with a word.
-    ///
-    /// See [`MetaItemParser::word`] for examples of words.
-    pub fn all_word_list<'s>(&'s self) -> Option<Vec<(Ident, &'s ArgParser<'a>)>> {
-        self.mixed().map(|i| i.meta_item()?.word()).collect()
-    }
-
-    /// Asserts that every item in the list is another list starting with a full path.
-    ///
-    /// See [`MetaItemParser::path`] for examples of paths.
-    pub fn all_path_list<'s>(&'s self) -> Option<Vec<(PathParser<'a>, &'s ArgParser<'a>)>> {
-        self.mixed().map(|i| Some(i.meta_item()?.path())).collect()
-    }
-
     /// Returns Some if the list contains only a single element.
     ///
     /// Inside the Some is the parser to parse this single element.
diff --git a/compiler/rustc_borrowck/src/constraints/mod.rs b/compiler/rustc_borrowck/src/constraints/mod.rs
index 514bbfe359b..99ddccabd15 100644
--- a/compiler/rustc_borrowck/src/constraints/mod.rs
+++ b/compiler/rustc_borrowck/src/constraints/mod.rs
@@ -5,11 +5,9 @@ use rustc_index::{IndexSlice, IndexVec};
 use rustc_middle::mir::ConstraintCategory;
 use rustc_middle::ty::{RegionVid, TyCtxt, VarianceDiagInfo};
 use rustc_span::Span;
-use tracing::{debug, instrument};
+use tracing::debug;
 
-use crate::region_infer::{AnnotatedSccs, ConstraintSccs, RegionDefinition, SccAnnotations};
 use crate::type_check::Locations;
-use crate::universal_regions::UniversalRegions;
 
 pub(crate) mod graph;
 
@@ -53,112 +51,6 @@ impl<'tcx> OutlivesConstraintSet<'tcx> {
     ) -> &IndexSlice<OutlivesConstraintIndex, OutlivesConstraint<'tcx>> {
         &self.outlives
     }
-
-    /// Computes cycles (SCCs) in the graph of regions. In particular,
-    /// find all regions R1, R2 such that R1: R2 and R2: R1 and group
-    /// them into an SCC, and find the relationships between SCCs.
-    pub(crate) fn compute_sccs(
-        &self,
-        static_region: RegionVid,
-        definitions: &IndexVec<RegionVid, RegionDefinition<'tcx>>,
-    ) -> AnnotatedSccs {
-        let constraint_graph = self.graph(definitions.len());
-        let region_graph = &constraint_graph.region_graph(self, static_region);
-        let mut annotation_visitor = SccAnnotations::new(definitions);
-        (
-            ConstraintSccs::new_with_annotation(&region_graph, &mut annotation_visitor),
-            annotation_visitor.scc_to_annotation,
-        )
-    }
-
-    /// This method handles Universe errors by rewriting the constraint
-    /// graph. For each strongly connected component in the constraint
-    /// graph such that there is a series of constraints
-    ///    A: B: C: ... : X  where
-    /// A's universe is smaller than X's and A is a placeholder,
-    /// add a constraint that A: 'static. This is a safe upper bound
-    /// in the face of borrow checker/trait solver limitations that will
-    /// eventually go away.
-    ///
-    /// For a more precise definition, see the documentation for
-    /// [`crate::region_infer::RegionTracker`].
-    ///
-    /// This edge case used to be handled during constraint propagation
-    /// by iterating over the strongly connected components in the constraint
-    /// graph while maintaining a set of bookkeeping mappings similar
-    /// to what is stored in `RegionTracker` and manually adding 'static as
-    /// needed.
-    ///
-    /// It was rewritten as part of the Polonius project with the goal of moving
-    /// higher-kindedness concerns out of the path of the borrow checker,
-    /// for two reasons:
-    ///
-    /// 1. Implementing Polonius is difficult enough without also
-    ///     handling them.
-    /// 2. The long-term goal is to handle higher-kinded concerns
-    ///     in the trait solver, where they belong. This avoids
-    ///     logic duplication and allows future trait solvers
-    ///     to compute better bounds than for example our
-    ///     "must outlive 'static" here.
-    ///
-    /// This code is a stop-gap measure in preparation for the future trait solver.
-    ///
-    /// Every constraint added by this method is an
-    /// internal `IllegalUniverse` constraint.
-    #[instrument(skip(self, universal_regions, definitions))]
-    pub(crate) fn add_outlives_static(
-        &mut self,
-        universal_regions: &UniversalRegions<'tcx>,
-        definitions: &IndexVec<RegionVid, RegionDefinition<'tcx>>,
-    ) -> AnnotatedSccs {
-        let fr_static = universal_regions.fr_static;
-        let (sccs, annotations) = self.compute_sccs(fr_static, definitions);
-
-        // Changed to `true` if we added any constraints to `self` and need to
-        // recompute SCCs.
-        let mut added_constraints = false;
-
-        for scc in sccs.all_sccs() {
-            // No point in adding 'static: 'static!
-            // This micro-optimisation makes somewhat sense
-            // because static outlives *everything*.
-            if scc == sccs.scc(fr_static) {
-                continue;
-            }
-
-            let annotation = annotations[scc];
-
-            // If this SCC participates in a universe violation,
-            // e.g. if it reaches a region with a universe smaller than
-            // the largest region reached, add a requirement that it must
-            // outlive `'static`.
-            if annotation.has_incompatible_universes() {
-                // Optimisation opportunity: this will add more constraints than
-                // needed for correctness, since an SCC upstream of another with
-                // a universe violation will "infect" its downstream SCCs to also
-                // outlive static.
-                added_constraints = true;
-                let scc_representative_outlives_static = OutlivesConstraint {
-                    sup: annotation.representative,
-                    sub: fr_static,
-                    category: ConstraintCategory::IllegalUniverse,
-                    locations: Locations::All(rustc_span::DUMMY_SP),
-                    span: rustc_span::DUMMY_SP,
-                    variance_info: VarianceDiagInfo::None,
-                    from_closure: false,
-                };
-                self.push(scc_representative_outlives_static);
-            }
-        }
-
-        if added_constraints {
-            // We changed the constraint set and so must recompute SCCs.
-            self.compute_sccs(fr_static, definitions)
-        } else {
-            // If we didn't add any back-edges; no more work needs doing
-            (sccs, annotations)
-        }
-    }
 }
 
 impl<'tcx> Index<OutlivesConstraintIndex> for OutlivesConstraintSet<'tcx> {
diff --git a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
index 3b7d31b1b13..1b4bb11d87b 100644
--- a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
@@ -263,7 +263,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                 // something that already has `Fn`-like bounds (or is a closure), so we can't
                 // restrict anyways.
             } else {
-                let copy_did = self.infcx.tcx.require_lang_item(LangItem::Copy, Some(span));
+                let copy_did = self.infcx.tcx.require_lang_item(LangItem::Copy, span);
                 self.suggest_adding_bounds(&mut err, ty, copy_did, span);
             }
 
@@ -1915,7 +1915,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
 
         let local_ty = self.body.local_decls[place.local].ty;
         let typeck_results = tcx.typeck(self.mir_def_id());
-        let clone = tcx.require_lang_item(LangItem::Clone, Some(body.span));
+        let clone = tcx.require_lang_item(LangItem::Clone, body.span);
         for expr in expr_finder.clones {
             if let hir::ExprKind::MethodCall(_, rcvr, _, span) = expr.kind
                 && let Some(rcvr_ty) = typeck_results.node_type_opt(rcvr.hir_id)
@@ -3314,7 +3314,13 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                     "function parameter".to_string(),
                     "function parameter borrowed here".to_string(),
                 ),
-                LocalKind::Temp if self.body.local_decls[local].is_user_variable() => {
+                LocalKind::Temp
+                    if self.body.local_decls[local].is_user_variable()
+                        && !self.body.local_decls[local]
+                            .source_info
+                            .span
+                            .in_external_macro(self.infcx.tcx.sess.source_map()) =>
+                {
                     ("local binding".to_string(), "local binding introduced here".to_string())
                 }
                 LocalKind::ReturnPointer | LocalKind::Temp => {
diff --git a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
index 56cc4327585..a5c9bad3ac2 100644
--- a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
@@ -840,14 +840,22 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
             } else {
                 bug!("not an upvar")
             };
-            err.span_label(
-                *span,
-                format!(
-                    "calling `{}` requires mutable binding due to {}",
-                    self.describe_place(the_place_err).unwrap(),
-                    reason
-                ),
-            );
+            // sometimes we deliberately don't store the name of a place when coming from a macro in
+            // another crate. We generally want to limit those diagnostics a little, to hide
+            // implementation details (such as those from pin!() or format!()). In that case show a
+            // slightly different error message, or none at all if something else happened. In other
+            // cases the message is likely not useful.
+            if let Some(place_name) = self.describe_place(the_place_err) {
+                err.span_label(
+                    *span,
+                    format!("calling `{place_name}` requires mutable binding due to {reason}"),
+                );
+            } else if span.from_expansion() {
+                err.span_label(
+                    *span,
+                    format!("a call in this macro requires a mutable binding due to {reason}",),
+                );
+            }
         }
     }
 
diff --git a/compiler/rustc_borrowck/src/diagnostics/region_name.rs b/compiler/rustc_borrowck/src/diagnostics/region_name.rs
index b08c10983bb..487f78058a8 100644
--- a/compiler/rustc_borrowck/src/diagnostics/region_name.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/region_name.rs
@@ -606,8 +606,8 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
         hir_args: &'hir hir::GenericArgs<'hir>,
         search_stack: &mut Vec<(Ty<'tcx>, &'hir hir::Ty<'hir>)>,
     ) -> Option<&'hir hir::Lifetime> {
-        for (kind, hir_arg) in iter::zip(args, hir_args.args) {
-            match (kind.unpack(), hir_arg) {
+        for (arg, hir_arg) in iter::zip(args, hir_args.args) {
+            match (arg.kind(), hir_arg) {
                 (GenericArgKind::Lifetime(r), hir::GenericArg::Lifetime(lt)) => {
                     if r.as_var() == needle_fr {
                         return Some(lt);
@@ -631,7 +631,7 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
                 ) => {
                     self.dcx().span_delayed_bug(
                         hir_arg.span(),
-                        format!("unmatched arg and hir arg: found {kind:?} vs {hir_arg:?}"),
+                        format!("unmatched arg and hir arg: found {arg:?} vs {hir_arg:?}"),
                     );
                 }
             }
@@ -997,7 +997,7 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
     ) -> bool {
         let tcx = self.infcx.tcx;
         ty.walk().any(|arg| {
-            if let ty::GenericArgKind::Type(ty) = arg.unpack()
+            if let ty::GenericArgKind::Type(ty) = arg.kind()
                 && let ty::Param(_) = ty.kind()
             {
                 clauses.iter().any(|pred| {
diff --git a/compiler/rustc_borrowck/src/handle_placeholders.rs b/compiler/rustc_borrowck/src/handle_placeholders.rs
new file mode 100644
index 00000000000..aaaf2f45c86
--- /dev/null
+++ b/compiler/rustc_borrowck/src/handle_placeholders.rs
@@ -0,0 +1,348 @@
+//! Logic for lowering higher-kinded outlives constraints
+//! (with placeholders and universes) and turn them into regular
+//! outlives constraints.
+
+use rustc_data_structures::frozen::Frozen;
+use rustc_data_structures::fx::FxIndexMap;
+use rustc_data_structures::graph::scc;
+use rustc_data_structures::graph::scc::Sccs;
+use rustc_index::IndexVec;
+use rustc_infer::infer::RegionVariableOrigin;
+use rustc_middle::mir::ConstraintCategory;
+use rustc_middle::ty::{RegionVid, UniverseIndex};
+use tracing::debug;
+
+use crate::constraints::{ConstraintSccIndex, OutlivesConstraintSet};
+use crate::consumers::OutlivesConstraint;
+use crate::diagnostics::UniverseInfo;
+use crate::member_constraints::MemberConstraintSet;
+use crate::region_infer::values::{LivenessValues, PlaceholderIndices};
+use crate::region_infer::{ConstraintSccs, RegionDefinition, Representative, TypeTest};
+use crate::ty::VarianceDiagInfo;
+use crate::type_check::free_region_relations::UniversalRegionRelations;
+use crate::type_check::{Locations, MirTypeckRegionConstraints};
+use crate::universal_regions::UniversalRegions;
+use crate::{BorrowckInferCtxt, NllRegionVariableOrigin};
+
+/// A set of outlives constraints after rewriting to remove
+/// higher-kinded constraints.
+pub(crate) struct LoweredConstraints<'tcx> {
+    pub(crate) constraint_sccs: Sccs<RegionVid, ConstraintSccIndex>,
+    pub(crate) definitions: Frozen<IndexVec<RegionVid, RegionDefinition<'tcx>>>,
+    pub(crate) scc_annotations: IndexVec<ConstraintSccIndex, RegionTracker>,
+    pub(crate) member_constraints: MemberConstraintSet<'tcx, RegionVid>,
+    pub(crate) outlives_constraints: Frozen<OutlivesConstraintSet<'tcx>>,
+    pub(crate) type_tests: Vec<TypeTest<'tcx>>,
+    pub(crate) liveness_constraints: LivenessValues,
+    pub(crate) universe_causes: FxIndexMap<UniverseIndex, UniverseInfo<'tcx>>,
+    pub(crate) placeholder_indices: PlaceholderIndices,
+}
+
+impl<'d, 'tcx, A: scc::Annotation> SccAnnotations<'d, 'tcx, A> {
+    pub(crate) fn init(definitions: &'d IndexVec<RegionVid, RegionDefinition<'tcx>>) -> Self {
+        Self { scc_to_annotation: IndexVec::new(), definitions }
+    }
+}
+
+/// A Visitor for SCC annotation construction.
+pub(crate) struct SccAnnotations<'d, 'tcx, A: scc::Annotation> {
+    pub(crate) scc_to_annotation: IndexVec<ConstraintSccIndex, A>,
+    definitions: &'d IndexVec<RegionVid, RegionDefinition<'tcx>>,
+}
+
+impl scc::Annotations<RegionVid> for SccAnnotations<'_, '_, RegionTracker> {
+    fn new(&self, element: RegionVid) -> RegionTracker {
+        RegionTracker::new(element, &self.definitions[element])
+    }
+
+    fn annotate_scc(&mut self, scc: ConstraintSccIndex, annotation: RegionTracker) {
+        let idx = self.scc_to_annotation.push(annotation);
+        assert!(idx == scc);
+    }
+
+    type Ann = RegionTracker;
+    type SccIdx = ConstraintSccIndex;
+}
+
+/// An annotation for region graph SCCs that tracks
+/// the values of its elements. This annotates a single SCC.
+#[derive(Copy, Debug, Clone)]
+pub(crate) struct RegionTracker {
+    /// The largest universe of a placeholder reached from this SCC.
+    /// This includes placeholders within this SCC.
+    max_placeholder_universe_reached: UniverseIndex,
+
+    /// The largest universe nameable from this SCC.
+    /// It is the smallest nameable universes of all
+    /// existential regions reachable from it.
+    max_nameable_universe: UniverseIndex,
+
+    /// The representative Region Variable Id for this SCC.
+    pub(crate) representative: Representative,
+}
+
+impl RegionTracker {
+    pub(crate) fn new(rvid: RegionVid, definition: &RegionDefinition<'_>) -> Self {
+        let placeholder_universe =
+            if matches!(definition.origin, NllRegionVariableOrigin::Placeholder(_)) {
+                definition.universe
+            } else {
+                UniverseIndex::ROOT
+            };
+
+        Self {
+            max_placeholder_universe_reached: placeholder_universe,
+            max_nameable_universe: definition.universe,
+            representative: Representative::new(rvid, definition),
+        }
+    }
+
+    /// The largest universe this SCC can name. It's the smallest
+    /// largest nameable uninverse of any reachable region.
+    pub(crate) fn max_nameable_universe(self) -> UniverseIndex {
+        self.max_nameable_universe
+    }
+
+    fn merge_min_max_seen(&mut self, other: &Self) {
+        self.max_placeholder_universe_reached = std::cmp::max(
+            self.max_placeholder_universe_reached,
+            other.max_placeholder_universe_reached,
+        );
+
+        self.max_nameable_universe =
+            std::cmp::min(self.max_nameable_universe, other.max_nameable_universe);
+    }
+
+    /// Returns `true` if during the annotated SCC reaches a placeholder
+    /// with a universe larger than the smallest nameable universe of any
+    /// reachable existential region.
+    pub(crate) fn has_incompatible_universes(&self) -> bool {
+        self.max_nameable_universe().cannot_name(self.max_placeholder_universe_reached)
+    }
+
+    /// Determine if the tracked universes of the two SCCs are compatible.
+    pub(crate) fn universe_compatible_with(&self, other: Self) -> bool {
+        self.max_nameable_universe().can_name(other.max_nameable_universe())
+            || self.max_nameable_universe().can_name(other.max_placeholder_universe_reached)
+    }
+}
+
+impl scc::Annotation for RegionTracker {
+    fn merge_scc(mut self, other: Self) -> Self {
+        self.representative = self.representative.merge_scc(other.representative);
+        self.merge_min_max_seen(&other);
+        self
+    }
+
+    fn merge_reached(mut self, other: Self) -> Self {
+        // No update to in-component values, only add seen values.
+        self.merge_min_max_seen(&other);
+        self
+    }
+}
+
+/// Determines if the region variable definitions contain
+/// placeholders, and compute them for later use.
+fn region_definitions<'tcx>(
+    universal_regions: &UniversalRegions<'tcx>,
+    infcx: &BorrowckInferCtxt<'tcx>,
+) -> (Frozen<IndexVec<RegionVid, RegionDefinition<'tcx>>>, bool) {
+    let var_infos = infcx.get_region_var_infos();
+    // Create a RegionDefinition for each inference variable. This happens here because
+    // it allows us to sneak in a cheap check for placeholders. Otherwise, its proper home
+    // is in `RegionInferenceContext::new()`, probably.
+    let mut definitions = IndexVec::with_capacity(var_infos.len());
+    let mut has_placeholders = false;
+
+    for info in var_infos.iter() {
+        let origin = match info.origin {
+            RegionVariableOrigin::Nll(origin) => origin,
+            _ => NllRegionVariableOrigin::Existential { from_forall: false },
+        };
+
+        let definition = RegionDefinition { origin, universe: info.universe, external_name: None };
+
+        has_placeholders |= matches!(origin, NllRegionVariableOrigin::Placeholder(_));
+        definitions.push(definition);
+    }
+
+    // Add external names from universal regions in fun function definitions.
+    // FIXME: this two-step method is annoying, but I don't know how to avoid it.
+    for (external_name, variable) in universal_regions.named_universal_regions_iter() {
+        debug!("region {:?} has external name {:?}", variable, external_name);
+        definitions[variable].external_name = Some(external_name);
+    }
+    (Frozen::freeze(definitions), has_placeholders)
+}
+
+/// This method handles placeholders by rewriting the constraint
+/// graph. For each strongly connected component in the constraint
+/// graph such that there is a series of constraints
+///    A: B: C: ... : X  where
+/// A contains a placeholder whose universe cannot be named by X,
+/// add a constraint that A: 'static. This is a safe upper bound
+/// in the face of borrow checker/trait solver limitations that will
+/// eventually go away.
+///
+/// For a more precise definition, see the documentation for
+/// [`RegionTracker`] and its methods!
+///
+/// This edge case used to be handled during constraint propagation.
+/// It was rewritten as part of the Polonius project with the goal of moving
+/// higher-kindedness concerns out of the path of the borrow checker,
+/// for two reasons:
+///
+/// 1. Implementing Polonius is difficult enough without also
+///     handling them.
+/// 2. The long-term goal is to handle higher-kinded concerns
+///     in the trait solver, where they belong. This avoids
+///     logic duplication and allows future trait solvers
+///     to compute better bounds than for example our
+///     "must outlive 'static" here.
+///
+/// This code is a stop-gap measure in preparation for the future trait solver.
+///
+/// Every constraint added by this method is an internal `IllegalUniverse` constraint.
+pub(crate) fn compute_sccs_applying_placeholder_outlives_constraints<'tcx>(
+    constraints: MirTypeckRegionConstraints<'tcx>,
+    universal_region_relations: &Frozen<UniversalRegionRelations<'tcx>>,
+    infcx: &BorrowckInferCtxt<'tcx>,
+) -> LoweredConstraints<'tcx> {
+    let universal_regions = &universal_region_relations.universal_regions;
+    let (definitions, has_placeholders) = region_definitions(universal_regions, infcx);
+
+    let MirTypeckRegionConstraints {
+        placeholder_indices,
+        placeholder_index_to_region: _,
+        liveness_constraints,
+        mut outlives_constraints,
+        mut member_constraints,
+        universe_causes,
+        type_tests,
+    } = constraints;
+
+    if let Some(guar) = universal_regions.tainted_by_errors() {
+        debug!("Universal regions tainted by errors; removing constraints!");
+        // Suppress unhelpful extra errors in `infer_opaque_types` by clearing out all
+        // outlives bounds that we may end up checking.
+        outlives_constraints = Default::default();
+        member_constraints = Default::default();
+
+        // Also taint the entire scope.
+        infcx.set_tainted_by_errors(guar);
+    }
+
+    let fr_static = universal_regions.fr_static;
+    let compute_sccs =
+        |constraints: &OutlivesConstraintSet<'tcx>,
+         annotations: &mut SccAnnotations<'_, 'tcx, RegionTracker>| {
+            ConstraintSccs::new_with_annotation(
+                &constraints.graph(definitions.len()).region_graph(constraints, fr_static),
+                annotations,
+            )
+        };
+
+    let mut scc_annotations = SccAnnotations::init(&definitions);
+    let constraint_sccs = compute_sccs(&outlives_constraints, &mut scc_annotations);
+
+    // This code structure is a bit convoluted because it allows for a planned
+    // future change where the early return here has a different type of annotation
+    // that does much less work.
+    if !has_placeholders {
+        debug!("No placeholder regions found; skipping rewriting logic!");
+
+        return LoweredConstraints {
+            type_tests,
+            member_constraints,
+            constraint_sccs,
+            scc_annotations: scc_annotations.scc_to_annotation,
+            definitions,
+            outlives_constraints: Frozen::freeze(outlives_constraints),
+            liveness_constraints,
+            universe_causes,
+            placeholder_indices,
+        };
+    }
+    debug!("Placeholders present; activating placeholder handling logic!");
+
+    let added_constraints = rewrite_placeholder_outlives(
+        &constraint_sccs,
+        &scc_annotations,
+        fr_static,
+        &mut outlives_constraints,
+    );
+
+    let (constraint_sccs, scc_annotations) = if added_constraints {
+        let mut annotations = SccAnnotations::init(&definitions);
+
+        // We changed the constraint set and so must recompute SCCs.
+        // Optimisation opportunity: if we can add them incrementally (and that's
+        // possible because edges to 'static always only merge SCCs into 'static),
+        // we would potentially save a lot of work here.
+        (compute_sccs(&outlives_constraints, &mut annotations), annotations.scc_to_annotation)
+    } else {
+        // If we didn't add any back-edges; no more work needs doing
+        debug!("No constraints rewritten!");
+        (constraint_sccs, scc_annotations.scc_to_annotation)
+    };
+
+    LoweredConstraints {
+        constraint_sccs,
+        definitions,
+        scc_annotations,
+        member_constraints,
+        outlives_constraints: Frozen::freeze(outlives_constraints),
+        type_tests,
+        liveness_constraints,
+        universe_causes,
+        placeholder_indices,
+    }
+}
+
+fn rewrite_placeholder_outlives<'tcx>(
+    sccs: &Sccs<RegionVid, ConstraintSccIndex>,
+    annotations: &SccAnnotations<'_, '_, RegionTracker>,
+    fr_static: RegionVid,
+    outlives_constraints: &mut OutlivesConstraintSet<'tcx>,
+) -> bool {
+    // Changed to `true` if we added any constraints and need to
+    // recompute SCCs.
+    let mut added_constraints = false;
+
+    let annotations = &annotations.scc_to_annotation;
+
+    for scc in sccs.all_sccs() {
+        // No point in adding 'static: 'static!
+        // This micro-optimisation makes somewhat sense
+        // because static outlives *everything*.
+        if scc == sccs.scc(fr_static) {
+            continue;
+        }
+
+        let annotation = annotations[scc];
+
+        // If this SCC participates in a universe violation,
+        // e.g. if it reaches a region with a universe smaller than
+        // the largest region reached, add a requirement that it must
+        // outlive `'static`.
+        if annotation.has_incompatible_universes() {
+            // Optimisation opportunity: this will add more constraints than
+            // needed for correctness, since an SCC upstream of another with
+            // a universe violation will "infect" its downstream SCCs to also
+            // outlive static.
+            let scc_representative_outlives_static = OutlivesConstraint {
+                sup: annotation.representative.rvid(),
+                sub: fr_static,
+                category: ConstraintCategory::IllegalUniverse,
+                locations: Locations::All(rustc_span::DUMMY_SP),
+                span: rustc_span::DUMMY_SP,
+                variance_info: VarianceDiagInfo::None,
+                from_closure: false,
+            };
+            outlives_constraints.push(scc_representative_outlives_static);
+            added_constraints = true;
+            debug!("Added {:?}: 'static!", annotation.representative.rvid());
+        }
+    }
+    added_constraints
+}
diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs
index 676cb618b72..e6eae7d4f5a 100644
--- a/compiler/rustc_borrowck/src/lib.rs
+++ b/compiler/rustc_borrowck/src/lib.rs
@@ -40,9 +40,7 @@ use rustc_middle::ty::{
     self, ParamEnv, RegionVid, Ty, TyCtxt, TypeFoldable, TypeVisitable, TypingMode, fold_regions,
 };
 use rustc_middle::{bug, span_bug};
-use rustc_mir_dataflow::impls::{
-    EverInitializedPlaces, MaybeInitializedPlaces, MaybeUninitializedPlaces,
-};
+use rustc_mir_dataflow::impls::{EverInitializedPlaces, MaybeUninitializedPlaces};
 use rustc_mir_dataflow::move_paths::{
     InitIndex, InitLocation, LookupResult, MoveData, MovePathIndex,
 };
@@ -74,6 +72,7 @@ mod constraints;
 mod dataflow;
 mod def_use;
 mod diagnostics;
+mod handle_placeholders;
 mod member_constraints;
 mod nll;
 mod path_utils;
@@ -324,10 +323,6 @@ fn do_mir_borrowck<'tcx>(
 
     let move_data = MoveData::gather_moves(body, tcx, |_| true);
 
-    let flow_inits = MaybeInitializedPlaces::new(tcx, body, &move_data)
-        .iterate_to_fixpoint(tcx, body, Some("borrowck"))
-        .into_results_cursor(body);
-
     let locals_are_invalidated_at_exit = tcx.hir_body_owner_kind(def).is_fn_or_closure();
     let borrow_set = BorrowSet::build(tcx, body, locals_are_invalidated_at_exit, &move_data);
 
@@ -346,7 +341,6 @@ fn do_mir_borrowck<'tcx>(
         body,
         &promoted,
         &location_table,
-        flow_inits,
         &move_data,
         &borrow_set,
         consumer_options,
diff --git a/compiler/rustc_borrowck/src/nll.rs b/compiler/rustc_borrowck/src/nll.rs
index fe899bb054f..1b011d73385 100644
--- a/compiler/rustc_borrowck/src/nll.rs
+++ b/compiler/rustc_borrowck/src/nll.rs
@@ -11,8 +11,6 @@ use rustc_middle::mir::pretty::{PrettyPrintMirOptions, dump_mir_with_options};
 use rustc_middle::mir::{Body, PassWhere, Promoted, create_dump_file, dump_enabled, dump_mir};
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::{self, TyCtxt};
-use rustc_mir_dataflow::ResultsCursor;
-use rustc_mir_dataflow::impls::MaybeInitializedPlaces;
 use rustc_mir_dataflow::move_paths::MoveData;
 use rustc_mir_dataflow::points::DenseLocationMap;
 use rustc_session::config::MirIncludeSpans;
@@ -22,6 +20,7 @@ use tracing::{debug, instrument};
 use crate::borrow_set::BorrowSet;
 use crate::consumers::ConsumerOptions;
 use crate::diagnostics::RegionErrors;
+use crate::handle_placeholders::compute_sccs_applying_placeholder_outlives_constraints;
 use crate::polonius::PoloniusDiagnosticsContext;
 use crate::polonius::legacy::{
     PoloniusFacts, PoloniusFactsExt, PoloniusLocationTable, PoloniusOutput,
@@ -75,14 +74,13 @@ pub(crate) fn replace_regions_in_mir<'tcx>(
 /// Computes the (non-lexical) regions from the input MIR.
 ///
 /// This may result in errors being reported.
-pub(crate) fn compute_regions<'a, 'tcx>(
+pub(crate) fn compute_regions<'tcx>(
     root_cx: &mut BorrowCheckRootCtxt<'tcx>,
     infcx: &BorrowckInferCtxt<'tcx>,
     universal_regions: UniversalRegions<'tcx>,
     body: &Body<'tcx>,
     promoted: &IndexSlice<Promoted, Body<'tcx>>,
     location_table: &PoloniusLocationTable,
-    flow_inits: ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>>,
     move_data: &MoveData<'tcx>,
     borrow_set: &BorrowSet<'tcx>,
     consumer_options: Option<ConsumerOptions>,
@@ -112,11 +110,16 @@ pub(crate) fn compute_regions<'a, 'tcx>(
         location_table,
         borrow_set,
         &mut polonius_facts,
-        flow_inits,
         move_data,
         Rc::clone(&location_map),
     );
 
+    let lowered_constraints = compute_sccs_applying_placeholder_outlives_constraints(
+        constraints,
+        &universal_region_relations,
+        infcx,
+    );
+
     // If requested, emit legacy polonius facts.
     polonius::legacy::emit_facts(
         &mut polonius_facts,
@@ -126,11 +129,15 @@ pub(crate) fn compute_regions<'a, 'tcx>(
         borrow_set,
         move_data,
         &universal_region_relations,
-        &constraints,
+        &lowered_constraints,
     );
 
-    let mut regioncx =
-        RegionInferenceContext::new(infcx, constraints, universal_region_relations, location_map);
+    let mut regioncx = RegionInferenceContext::new(
+        infcx,
+        lowered_constraints,
+        universal_region_relations,
+        location_map,
+    );
 
     // If requested for `-Zpolonius=next`, convert NLL constraints to localized outlives constraints
     // and use them to compute loan liveness.
diff --git a/compiler/rustc_borrowck/src/polonius/legacy/mod.rs b/compiler/rustc_borrowck/src/polonius/legacy/mod.rs
index 95820c07a02..05fd6e39476 100644
--- a/compiler/rustc_borrowck/src/polonius/legacy/mod.rs
+++ b/compiler/rustc_borrowck/src/polonius/legacy/mod.rs
@@ -13,7 +13,7 @@ use tracing::debug;
 
 use crate::borrow_set::BorrowSet;
 use crate::constraints::OutlivesConstraint;
-use crate::type_check::MirTypeckRegionConstraints;
+use crate::handle_placeholders::LoweredConstraints;
 use crate::type_check::free_region_relations::UniversalRegionRelations;
 use crate::universal_regions::UniversalRegions;
 
@@ -43,7 +43,7 @@ pub(crate) fn emit_facts<'tcx>(
     borrow_set: &BorrowSet<'tcx>,
     move_data: &MoveData<'tcx>,
     universal_region_relations: &UniversalRegionRelations<'tcx>,
-    constraints: &MirTypeckRegionConstraints<'tcx>,
+    constraints: &LoweredConstraints<'tcx>,
 ) {
     let Some(facts) = facts else {
         // We don't do anything if there are no facts to fill.
@@ -203,7 +203,7 @@ pub(crate) fn emit_drop_facts<'tcx>(
 fn emit_outlives_facts<'tcx>(
     facts: &mut PoloniusFacts,
     location_table: &PoloniusLocationTable,
-    constraints: &MirTypeckRegionConstraints<'tcx>,
+    constraints: &LoweredConstraints<'tcx>,
 ) {
     facts.subset_base.extend(constraints.outlives_constraints.outlives().iter().flat_map(
         |constraint: &OutlivesConstraint<'_>| {
diff --git a/compiler/rustc_borrowck/src/region_infer/dump_mir.rs b/compiler/rustc_borrowck/src/region_infer/dump_mir.rs
index ef3d6309c19..a9ab30fd8fa 100644
--- a/compiler/rustc_borrowck/src/region_infer/dump_mir.rs
+++ b/compiler/rustc_borrowck/src/region_infer/dump_mir.rs
@@ -46,7 +46,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
                 "| {r:rw$?} | {ui:4?} | {v}",
                 r = region,
                 rw = REGION_WIDTH,
-                ui = self.region_universe(region),
+                ui = self.max_nameable_universe(self.constraint_sccs.scc(region)),
                 v = self.region_value_str(region),
             )?;
         }
diff --git a/compiler/rustc_borrowck/src/region_infer/mod.rs b/compiler/rustc_borrowck/src/region_infer/mod.rs
index b4ff3d66f3d..5f1b655c6b6 100644
--- a/compiler/rustc_borrowck/src/region_infer/mod.rs
+++ b/compiler/rustc_borrowck/src/region_infer/mod.rs
@@ -1,3 +1,4 @@
+use std::cell::OnceCell;
 use std::collections::VecDeque;
 use std::rc::Rc;
 
@@ -10,7 +11,7 @@ use rustc_hir::def_id::CRATE_DEF_ID;
 use rustc_index::IndexVec;
 use rustc_infer::infer::outlives::test_type_match;
 use rustc_infer::infer::region_constraints::{GenericKind, VerifyBound, VerifyIfEq};
-use rustc_infer::infer::{InferCtxt, NllRegionVariableOrigin, RegionVariableOrigin};
+use rustc_infer::infer::{InferCtxt, NllRegionVariableOrigin};
 use rustc_middle::bug;
 use rustc_middle::mir::{
     AnnotationSource, BasicBlock, Body, ConstraintCategory, Local, Location, ReturnConstraint,
@@ -27,13 +28,14 @@ use crate::constraints::graph::{self, NormalConstraintGraph, RegionGraph};
 use crate::constraints::{ConstraintSccIndex, OutlivesConstraint, OutlivesConstraintSet};
 use crate::dataflow::BorrowIndex;
 use crate::diagnostics::{RegionErrorKind, RegionErrors, UniverseInfo};
+use crate::handle_placeholders::{LoweredConstraints, RegionTracker};
 use crate::member_constraints::{MemberConstraintSet, NllMemberConstraintIndex};
 use crate::polonius::LiveLoans;
 use crate::polonius::legacy::PoloniusOutput;
 use crate::region_infer::reverse_sccs::ReverseSccGraph;
 use crate::region_infer::values::{LivenessValues, RegionElement, RegionValues, ToElementIndex};
+use crate::type_check::Locations;
 use crate::type_check::free_region_relations::UniversalRegionRelations;
-use crate::type_check::{Locations, MirTypeckRegionConstraints};
 use crate::universal_regions::UniversalRegions;
 use crate::{
     BorrowckInferCtxt, ClosureOutlivesRequirement, ClosureOutlivesSubject,
@@ -47,125 +49,48 @@ mod reverse_sccs;
 
 pub(crate) mod values;
 
-pub(crate) type ConstraintSccs = Sccs<RegionVid, ConstraintSccIndex>;
-pub(crate) type AnnotatedSccs = (ConstraintSccs, IndexVec<ConstraintSccIndex, RegionTracker>);
-
-/// An annotation for region graph SCCs that tracks
-/// the values of its elements. This annotates a single SCC.
-#[derive(Copy, Debug, Clone)]
-pub(crate) struct RegionTracker {
-    /// The largest universe of a placeholder reached from this SCC.
-    /// This includes placeholders within this SCC.
-    max_placeholder_universe_reached: UniverseIndex,
-
-    /// The smallest universe index reachable form the nodes of this SCC.
-    min_reachable_universe: UniverseIndex,
-
-    /// The representative Region Variable Id for this SCC. We prefer
-    /// placeholders over existentially quantified variables, otherwise
-    ///  it's the one with the smallest Region Variable ID.
-    pub(crate) representative: RegionVid,
-
-    /// Is the current representative a placeholder?
-    representative_is_placeholder: bool,
-
-    /// Is the current representative existentially quantified?
-    representative_is_existential: bool,
+/// The representative region variable for an SCC, tagged by its origin.
+/// We prefer placeholders over existentially quantified variables, otherwise
+/// it's the one with the smallest Region Variable ID. In other words,
+/// the order of this enumeration really matters!
+#[derive(Copy, Debug, Clone, PartialEq, PartialOrd, Eq, Ord)]
+pub(crate) enum Representative {
+    FreeRegion(RegionVid),
+    Placeholder(RegionVid),
+    Existential(RegionVid),
 }
 
-impl scc::Annotation for RegionTracker {
-    fn merge_scc(mut self, mut other: Self) -> Self {
-        // Prefer any placeholder over any existential
-        if other.representative_is_placeholder && self.representative_is_existential {
-            other.merge_min_max_seen(&self);
-            return other;
-        }
-
-        if self.representative_is_placeholder && other.representative_is_existential
-            || (self.representative <= other.representative)
-        {
-            self.merge_min_max_seen(&other);
-            return self;
+impl Representative {
+    pub(crate) fn rvid(self) -> RegionVid {
+        match self {
+            Representative::FreeRegion(region_vid)
+            | Representative::Placeholder(region_vid)
+            | Representative::Existential(region_vid) => region_vid,
         }
-        other.merge_min_max_seen(&self);
-        other
-    }
-
-    fn merge_reached(mut self, other: Self) -> Self {
-        // No update to in-component values, only add seen values.
-        self.merge_min_max_seen(&other);
-        self
     }
-}
-
-/// A Visitor for SCC annotation construction.
-pub(crate) struct SccAnnotations<'d, 'tcx, A: scc::Annotation> {
-    pub(crate) scc_to_annotation: IndexVec<ConstraintSccIndex, A>,
-    definitions: &'d IndexVec<RegionVid, RegionDefinition<'tcx>>,
-}
 
-impl<'d, 'tcx, A: scc::Annotation> SccAnnotations<'d, 'tcx, A> {
-    pub(crate) fn new(definitions: &'d IndexVec<RegionVid, RegionDefinition<'tcx>>) -> Self {
-        Self { scc_to_annotation: IndexVec::new(), definitions }
+    pub(crate) fn new(r: RegionVid, definition: &RegionDefinition<'_>) -> Self {
+        match definition.origin {
+            NllRegionVariableOrigin::FreeRegion => Representative::FreeRegion(r),
+            NllRegionVariableOrigin::Placeholder(_) => Representative::Placeholder(r),
+            NllRegionVariableOrigin::Existential { .. } => Representative::Existential(r),
+        }
     }
 }
 
-impl scc::Annotations<RegionVid> for SccAnnotations<'_, '_, RegionTracker> {
-    fn new(&self, element: RegionVid) -> RegionTracker {
-        RegionTracker::new(element, &self.definitions[element])
+impl scc::Annotation for Representative {
+    fn merge_scc(self, other: Self) -> Self {
+        // Just pick the smallest one. Note that we order by tag first!
+        std::cmp::min(self, other)
     }
 
-    fn annotate_scc(&mut self, scc: ConstraintSccIndex, annotation: RegionTracker) {
-        let idx = self.scc_to_annotation.push(annotation);
-        assert!(idx == scc);
+    // For reachability, we do nothing since the representative doesn't change.
+    fn merge_reached(self, _other: Self) -> Self {
+        self
     }
-
-    type Ann = RegionTracker;
-    type SccIdx = ConstraintSccIndex;
 }
 
-impl RegionTracker {
-    pub(crate) fn new(rvid: RegionVid, definition: &RegionDefinition<'_>) -> Self {
-        let (representative_is_placeholder, representative_is_existential) = match definition.origin
-        {
-            NllRegionVariableOrigin::FreeRegion => (false, false),
-            NllRegionVariableOrigin::Placeholder(_) => (true, false),
-            NllRegionVariableOrigin::Existential { .. } => (false, true),
-        };
-
-        let placeholder_universe =
-            if representative_is_placeholder { definition.universe } else { UniverseIndex::ROOT };
-
-        Self {
-            max_placeholder_universe_reached: placeholder_universe,
-            min_reachable_universe: definition.universe,
-            representative: rvid,
-            representative_is_placeholder,
-            representative_is_existential,
-        }
-    }
-
-    /// The smallest-indexed universe reachable from and/or in this SCC.
-    fn min_universe(self) -> UniverseIndex {
-        self.min_reachable_universe
-    }
-
-    fn merge_min_max_seen(&mut self, other: &Self) {
-        self.max_placeholder_universe_reached = std::cmp::max(
-            self.max_placeholder_universe_reached,
-            other.max_placeholder_universe_reached,
-        );
-
-        self.min_reachable_universe =
-            std::cmp::min(self.min_reachable_universe, other.min_reachable_universe);
-    }
-
-    /// Returns `true` if during the annotated SCC reaches a placeholder
-    /// with a universe larger than the smallest reachable one, `false` otherwise.
-    pub(crate) fn has_incompatible_universes(&self) -> bool {
-        self.min_universe().cannot_name(self.max_placeholder_universe_reached)
-    }
-}
+pub(crate) type ConstraintSccs = Sccs<RegionVid, ConstraintSccIndex>;
 
 pub struct RegionInferenceContext<'tcx> {
     /// Contains the definition for every region variable. Region
@@ -197,8 +122,8 @@ pub struct RegionInferenceContext<'tcx> {
 
     /// Reverse of the SCC constraint graph --  i.e., an edge `A -> B` exists if
     /// `B: A`. This is used to compute the universal regions that are required
-    /// to outlive a given SCC. Computed lazily.
-    rev_scc_graph: Option<ReverseSccGraph>,
+    /// to outlive a given SCC.
+    rev_scc_graph: OnceCell<ReverseSccGraph>,
 
     /// The "R0 member of [R1..Rn]" constraints, indexed by SCC.
     member_constraints: Rc<MemberConstraintSet<'tcx, ConstraintSccIndex>>,
@@ -413,26 +338,6 @@ fn sccs_info<'tcx>(infcx: &BorrowckInferCtxt<'tcx>, sccs: &ConstraintSccs) {
     debug!("SCC edges {:#?}", scc_node_to_edges);
 }
 
-fn create_definitions<'tcx>(
-    infcx: &BorrowckInferCtxt<'tcx>,
-    universal_regions: &UniversalRegions<'tcx>,
-) -> Frozen<IndexVec<RegionVid, RegionDefinition<'tcx>>> {
-    // Create a RegionDefinition for each inference variable.
-    let mut definitions: IndexVec<_, _> = infcx
-        .get_region_var_infos()
-        .iter()
-        .map(|info| RegionDefinition::new(info.universe, info.origin))
-        .collect();
-
-    // Add the external name for all universal regions.
-    for (external_name, variable) in universal_regions.named_universal_regions_iter() {
-        debug!("region {variable:?} has external name {external_name:?}");
-        definitions[variable].external_name = Some(external_name);
-    }
-
-    Frozen::freeze(definitions)
-}
-
 impl<'tcx> RegionInferenceContext<'tcx> {
     /// Creates a new region inference context with a total of
     /// `num_region_variables` valid inference variables; the first N
@@ -443,42 +348,30 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     /// of constraints produced by the MIR type check.
     pub(crate) fn new(
         infcx: &BorrowckInferCtxt<'tcx>,
-        constraints: MirTypeckRegionConstraints<'tcx>,
+        lowered_constraints: LoweredConstraints<'tcx>,
         universal_region_relations: Frozen<UniversalRegionRelations<'tcx>>,
         location_map: Rc<DenseLocationMap>,
     ) -> Self {
         let universal_regions = &universal_region_relations.universal_regions;
-        let MirTypeckRegionConstraints {
-            placeholder_indices,
-            placeholder_index_to_region: _,
+
+        let LoweredConstraints {
+            constraint_sccs,
+            definitions,
+            outlives_constraints,
+            scc_annotations,
+            type_tests,
             liveness_constraints,
-            mut outlives_constraints,
-            mut member_constraints,
             universe_causes,
-            type_tests,
-        } = constraints;
+            placeholder_indices,
+            member_constraints,
+        } = lowered_constraints;
 
         debug!("universal_regions: {:#?}", universal_region_relations.universal_regions);
         debug!("outlives constraints: {:#?}", outlives_constraints);
         debug!("placeholder_indices: {:#?}", placeholder_indices);
         debug!("type tests: {:#?}", type_tests);
 
-        if let Some(guar) = universal_region_relations.universal_regions.tainted_by_errors() {
-            // Suppress unhelpful extra errors in `infer_opaque_types` by clearing out all
-            // outlives bounds that we may end up checking.
-            outlives_constraints = Default::default();
-            member_constraints = Default::default();
-
-            // Also taint the entire scope.
-            infcx.set_tainted_by_errors(guar);
-        }
-
-        let definitions = create_definitions(infcx, &universal_regions);
-
-        let (constraint_sccs, scc_annotations) =
-            outlives_constraints.add_outlives_static(&universal_regions, &definitions);
-        let constraints = Frozen::freeze(outlives_constraints);
-        let constraint_graph = Frozen::freeze(constraints.graph(definitions.len()));
+        let constraint_graph = Frozen::freeze(outlives_constraints.graph(definitions.len()));
 
         if cfg!(debug_assertions) {
             sccs_info(infcx, &constraint_sccs);
@@ -498,11 +391,11 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         let mut result = Self {
             definitions,
             liveness_constraints,
-            constraints,
+            constraints: outlives_constraints,
             constraint_graph,
             constraint_sccs,
             scc_annotations,
-            rev_scc_graph: None,
+            rev_scc_graph: OnceCell::new(),
             member_constraints,
             member_constraints_applied: Vec::new(),
             universe_causes,
@@ -657,11 +550,6 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         self.scc_values.placeholders_contained_in(scc)
     }
 
-    /// Returns access to the value of `r` for debugging purposes.
-    pub(crate) fn region_universe(&self, r: RegionVid) -> ty::UniverseIndex {
-        self.scc_universe(self.constraint_sccs.scc(r))
-    }
-
     /// Once region solving has completed, this function will return the member constraints that
     /// were applied to the value of a given SCC `scc`. See `AppliedMemberConstraint`.
     pub(crate) fn applied_member_constraints(
@@ -809,9 +697,6 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         member_constraint_index: NllMemberConstraintIndex,
         choice_regions: &[ty::RegionVid],
     ) {
-        // Lazily compute the reverse graph, we'll need it later.
-        self.compute_reverse_scc_graph();
-
         // Create a mutable vector of the options. We'll try to winnow
         // them down.
         let mut choice_regions: Vec<ty::RegionVid> = choice_regions.to_vec();
@@ -828,7 +713,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
 
         // If the member region lives in a higher universe, we currently choose
         // the most conservative option by leaving it unchanged.
-        if !self.scc_universe(scc).is_root() {
+        if !self.max_nameable_universe(scc).is_root() {
             return;
         }
 
@@ -849,7 +734,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         // R0`). Therefore, we need only keep an option `O` if `UB: O`
         // for all UB.
         let universal_region_relations = &self.universal_region_relations;
-        for ub in self.rev_scc_graph.as_ref().unwrap().upper_bounds(scc) {
+        for ub in self.reverse_scc_graph().upper_bounds(scc) {
             debug!(?ub);
             choice_regions.retain(|&o_r| universal_region_relations.outlives(ub, o_r));
         }
@@ -904,20 +789,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     /// in `scc_a`. Used during constraint propagation, and only once
     /// the value of `scc_b` has been computed.
     fn universe_compatible(&self, scc_b: ConstraintSccIndex, scc_a: ConstraintSccIndex) -> bool {
-        let a_annotation = self.scc_annotations[scc_a];
-        let b_annotation = self.scc_annotations[scc_b];
-        let a_universe = a_annotation.min_universe();
-
-        // If scc_b's declared universe is a subset of
-        // scc_a's declared universe (typically, both are ROOT), then
-        // it cannot contain any problematic universe elements.
-        if a_universe.can_name(b_annotation.min_universe()) {
-            return true;
-        }
-
-        // Otherwise, there can be no placeholder in `b` with a too high
-        // universe index to name from `a`.
-        a_universe.can_name(b_annotation.max_placeholder_universe_reached)
+        self.scc_annotations[scc_a].universe_compatible_with(self.scc_annotations[scc_b])
     }
 
     /// Once regions have been propagated, this method is used to see
@@ -1021,7 +893,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
             "lower_bound = {:?} r_scc={:?} universe={:?}",
             lower_bound,
             r_scc,
-            self.scc_universe(r_scc)
+            self.max_nameable_universe(r_scc)
         );
         // If the type test requires that `T: 'a` where `'a` is a
         // placeholder from another universe, that effectively requires
@@ -1499,10 +1371,9 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         }
     }
 
-    /// The minimum universe of any variable reachable from this
-    /// SCC, inside or outside of it.
-    fn scc_universe(&self, scc: ConstraintSccIndex) -> UniverseIndex {
-        self.scc_annotations[scc].min_universe()
+    /// The largest universe of any region nameable from this SCC.
+    fn max_nameable_universe(&self, scc: ConstraintSccIndex) -> UniverseIndex {
+        self.scc_annotations[scc].max_nameable_universe()
     }
 
     /// Checks the final value for the free region `fr` to see if it
@@ -1524,7 +1395,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
 
         // Because this free region must be in the ROOT universe, we
         // know it cannot contain any bound universes.
-        assert!(self.scc_universe(longer_fr_scc).is_root());
+        assert!(self.max_nameable_universe(longer_fr_scc).is_root());
 
         // Only check all of the relations for the main representative of each
         // SCC, otherwise just check that we outlive said representative. This
@@ -1915,7 +1786,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     #[instrument(skip(self), level = "trace", ret)]
     pub(crate) fn find_sub_region_live_at(&self, fr1: RegionVid, location: Location) -> RegionVid {
         trace!(scc = ?self.constraint_sccs.scc(fr1));
-        trace!(universe = ?self.region_universe(fr1));
+        trace!(universe = ?self.max_nameable_universe(self.constraint_sccs.scc(fr1)));
         self.find_constraint_paths_between_regions(fr1, |r| {
             // First look for some `r` such that `fr1: r` and `r` is live at `location`
             trace!(?r, liveness_constraints=?self.liveness_constraints.pretty_print_live_points(r));
@@ -2246,7 +2117,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     /// they *must* be equal (though not having the same repr does not
     /// mean they are unequal).
     fn scc_representative(&self, scc: ConstraintSccIndex) -> RegionVid {
-        self.scc_annotations[scc].representative
+        self.scc_annotations[scc].representative.rvid()
     }
 
     pub(crate) fn liveness_constraints(&self) -> &LivenessValues {
@@ -2268,21 +2139,6 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     }
 }
 
-impl<'tcx> RegionDefinition<'tcx> {
-    fn new(universe: ty::UniverseIndex, rv_origin: RegionVariableOrigin) -> Self {
-        // Create a new region definition. Note that, for free
-        // regions, the `external_name` field gets updated later in
-        // `init_free_and_bound_regions`.
-
-        let origin = match rv_origin {
-            RegionVariableOrigin::Nll(origin) => origin,
-            _ => NllRegionVariableOrigin::Existential { from_forall: false },
-        };
-
-        Self { origin, universe, external_name: None }
-    }
-}
-
 #[derive(Clone, Debug)]
 pub(crate) struct BlameConstraint<'tcx> {
     pub category: ConstraintCategory<'tcx>,
diff --git a/compiler/rustc_borrowck/src/region_infer/opaque_types.rs b/compiler/rustc_borrowck/src/region_infer/opaque_types.rs
index 25cbd579ea1..6270e6d9a60 100644
--- a/compiler/rustc_borrowck/src/region_infer/opaque_types.rs
+++ b/compiler/rustc_borrowck/src/region_infer/opaque_types.rs
@@ -191,7 +191,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
                 let scc = self.constraint_sccs.scc(vid);
 
                 // Special handling of higher-ranked regions.
-                if !self.scc_universe(scc).is_root() {
+                if !self.max_nameable_universe(scc).is_root() {
                     match self.scc_values.placeholders_contained_in(scc).enumerate().last() {
                         // If the region contains a single placeholder then they're equal.
                         Some((0, placeholder)) => {
@@ -215,9 +215,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
                 // FIXME: We could probably compute the LUB if there is one.
                 let scc = self.constraint_sccs.scc(vid);
                 let upper_bounds: Vec<_> = self
-                    .rev_scc_graph
-                    .as_ref()
-                    .unwrap()
+                    .reverse_scc_graph()
                     .upper_bounds(scc)
                     .filter_map(|vid| self.definitions[vid].external_name)
                     .filter(|r| !r.is_static())
diff --git a/compiler/rustc_borrowck/src/region_infer/reverse_sccs.rs b/compiler/rustc_borrowck/src/region_infer/reverse_sccs.rs
index 8e04791461b..604265f8940 100644
--- a/compiler/rustc_borrowck/src/region_infer/reverse_sccs.rs
+++ b/compiler/rustc_borrowck/src/region_infer/reverse_sccs.rs
@@ -59,13 +59,10 @@ impl ReverseSccGraph {
 }
 
 impl RegionInferenceContext<'_> {
-    /// Compute the reverse SCC-based constraint graph (lazily).
-    pub(super) fn compute_reverse_scc_graph(&mut self) {
-        if self.rev_scc_graph.is_some() {
-            return;
-        }
-
-        self.rev_scc_graph =
-            Some(ReverseSccGraph::compute(&self.constraint_sccs, self.universal_regions()));
+    /// Return the reverse graph of the region SCCs, initialising it if needed.
+    pub(super) fn reverse_scc_graph(&self) -> &ReverseSccGraph {
+        self.rev_scc_graph.get_or_init(|| {
+            ReverseSccGraph::compute(&self.constraint_sccs, self.universal_regions())
+        })
     }
 }
diff --git a/compiler/rustc_borrowck/src/type_check/constraint_conversion.rs b/compiler/rustc_borrowck/src/type_check/constraint_conversion.rs
index 57516565147..a1c74672157 100644
--- a/compiler/rustc_borrowck/src/type_check/constraint_conversion.rs
+++ b/compiler/rustc_borrowck/src/type_check/constraint_conversion.rs
@@ -148,7 +148,7 @@ impl<'a, 'tcx> ConstraintConversion<'a, 'tcx> {
 
             let mut next_outlives_predicates = vec![];
             for (ty::OutlivesPredicate(k1, r2), constraint_category) in outlives_predicates {
-                match k1.unpack() {
+                match k1.kind() {
                     GenericArgKind::Lifetime(r1) => {
                         let r1_vid = self.to_region_vid(r1);
                         let r2_vid = self.to_region_vid(r2);
diff --git a/compiler/rustc_borrowck/src/type_check/free_region_relations.rs b/compiler/rustc_borrowck/src/type_check/free_region_relations.rs
index 92732aba29b..f642d34ea67 100644
--- a/compiler/rustc_borrowck/src/type_check/free_region_relations.rs
+++ b/compiler/rustc_borrowck/src/type_check/free_region_relations.rs
@@ -131,7 +131,8 @@ impl UniversalRegionRelations<'_> {
         assert!(self.universal_regions.is_universal_region(fr0));
 
         let mut external_parents = vec![];
-        let mut queue = vec![fr0];
+
+        let mut queue = vec![relation.minimal_scc_representative(fr0)];
 
         // Keep expanding `fr` into its parents until we reach
         // non-local regions.
diff --git a/compiler/rustc_borrowck/src/type_check/input_output.rs b/compiler/rustc_borrowck/src/type_check/input_output.rs
index c6b29fe36fd..0c46e0c0c22 100644
--- a/compiler/rustc_borrowck/src/type_check/input_output.rs
+++ b/compiler/rustc_borrowck/src/type_check/input_output.rs
@@ -52,7 +52,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             assert_matches!(
                 self.tcx().coroutine_kind(self.tcx().coroutine_for_closure(mir_def_id)),
                 Some(hir::CoroutineKind::Desugared(
-                    hir::CoroutineDesugaring::Async,
+                    hir::CoroutineDesugaring::Async | hir::CoroutineDesugaring::Gen,
                     hir::CoroutineSource::Closure
                 )),
                 "this needs to be modified if we're lowering non-async closures"
diff --git a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
index b7a21cf48c8..ca1b850f766 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
@@ -5,8 +5,6 @@ use rustc_middle::mir::{Body, Local, Location, SourceInfo};
 use rustc_middle::span_bug;
 use rustc_middle::ty::relate::Relate;
 use rustc_middle::ty::{GenericArgsRef, Region, RegionVid, Ty, TyCtxt, TypeVisitable};
-use rustc_mir_dataflow::ResultsCursor;
-use rustc_mir_dataflow::impls::MaybeInitializedPlaces;
 use rustc_mir_dataflow::move_paths::MoveData;
 use rustc_mir_dataflow::points::DenseLocationMap;
 use tracing::debug;
@@ -28,10 +26,9 @@ mod trace;
 ///
 /// N.B., this computation requires normalization; therefore, it must be
 /// performed before
-pub(super) fn generate<'a, 'tcx>(
+pub(super) fn generate<'tcx>(
     typeck: &mut TypeChecker<'_, 'tcx>,
     location_map: &DenseLocationMap,
-    flow_inits: ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>>,
     move_data: &MoveData<'tcx>,
 ) {
     debug!("liveness::generate");
@@ -58,7 +55,7 @@ pub(super) fn generate<'a, 'tcx>(
     let (relevant_live_locals, boring_locals) =
         compute_relevant_live_locals(typeck.tcx(), &free_regions, typeck.body);
 
-    trace::trace(typeck, location_map, flow_inits, move_data, relevant_live_locals, boring_locals);
+    trace::trace(typeck, location_map, move_data, relevant_live_locals, boring_locals);
 
     // Mark regions that should be live where they appear within rvalues or within a call: like
     // args, regions, and types.
diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
index 512288a0f7d..5d30fa71e92 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
@@ -7,10 +7,10 @@ use rustc_middle::mir::{BasicBlock, Body, ConstraintCategory, HasLocalDecls, Loc
 use rustc_middle::traits::query::DropckOutlivesResult;
 use rustc_middle::ty::relate::Relate;
 use rustc_middle::ty::{Ty, TyCtxt, TypeVisitable, TypeVisitableExt};
-use rustc_mir_dataflow::ResultsCursor;
 use rustc_mir_dataflow::impls::MaybeInitializedPlaces;
 use rustc_mir_dataflow::move_paths::{HasMoveData, MoveData, MovePathIndex};
 use rustc_mir_dataflow::points::{DenseLocationMap, PointIndex};
+use rustc_mir_dataflow::{Analysis, ResultsCursor};
 use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span};
 use rustc_trait_selection::error_reporting::InferCtxtErrorExt;
 use rustc_trait_selection::traits::ObligationCtxt;
@@ -37,10 +37,9 @@ use crate::type_check::{NormalizeLocation, TypeChecker};
 /// DROP-LIVE set are to the liveness sets for regions found in the
 /// `dropck_outlives` result of the variable's type (in particular,
 /// this respects `#[may_dangle]` annotations).
-pub(super) fn trace<'a, 'tcx>(
+pub(super) fn trace<'tcx>(
     typeck: &mut TypeChecker<'_, 'tcx>,
     location_map: &DenseLocationMap,
-    flow_inits: ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>>,
     move_data: &MoveData<'tcx>,
     relevant_live_locals: Vec<Local>,
     boring_locals: Vec<Local>,
@@ -48,7 +47,7 @@ pub(super) fn trace<'a, 'tcx>(
     let local_use_map = &LocalUseMap::build(&relevant_live_locals, location_map, typeck.body);
     let cx = LivenessContext {
         typeck,
-        flow_inits,
+        flow_inits: None,
         location_map,
         local_use_map,
         move_data,
@@ -65,7 +64,7 @@ pub(super) fn trace<'a, 'tcx>(
 }
 
 /// Contextual state for the type-liveness coroutine.
-struct LivenessContext<'a, 'typeck, 'b, 'tcx> {
+struct LivenessContext<'a, 'typeck, 'tcx> {
     /// Current type-checker, giving us our inference context etc.
     ///
     /// This also stores the body we're currently analyzing.
@@ -81,8 +80,8 @@ struct LivenessContext<'a, 'typeck, 'b, 'tcx> {
     drop_data: FxIndexMap<Ty<'tcx>, DropData<'tcx>>,
 
     /// Results of dataflow tracking which variables (and paths) have been
-    /// initialized.
-    flow_inits: ResultsCursor<'b, 'tcx, MaybeInitializedPlaces<'b, 'tcx>>,
+    /// initialized. Computed lazily when needed by drop-liveness.
+    flow_inits: Option<ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>>>,
 
     /// Index indicating where each variable is assigned, used, or
     /// dropped.
@@ -94,8 +93,8 @@ struct DropData<'tcx> {
     region_constraint_data: Option<&'tcx QueryRegionConstraints<'tcx>>,
 }
 
-struct LivenessResults<'a, 'typeck, 'b, 'tcx> {
-    cx: LivenessContext<'a, 'typeck, 'b, 'tcx>,
+struct LivenessResults<'a, 'typeck, 'tcx> {
+    cx: LivenessContext<'a, 'typeck, 'tcx>,
 
     /// Set of points that define the current local.
     defs: DenseBitSet<PointIndex>,
@@ -116,8 +115,8 @@ struct LivenessResults<'a, 'typeck, 'b, 'tcx> {
     stack: Vec<PointIndex>,
 }
 
-impl<'a, 'typeck, 'b, 'tcx> LivenessResults<'a, 'typeck, 'b, 'tcx> {
-    fn new(cx: LivenessContext<'a, 'typeck, 'b, 'tcx>) -> Self {
+impl<'a, 'typeck, 'tcx> LivenessResults<'a, 'typeck, 'tcx> {
+    fn new(cx: LivenessContext<'a, 'typeck, 'tcx>) -> Self {
         let num_points = cx.location_map.num_points();
         LivenessResults {
             cx,
@@ -459,20 +458,56 @@ impl<'a, 'typeck, 'b, 'tcx> LivenessResults<'a, 'typeck, 'b, 'tcx> {
     }
 }
 
-impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
+impl<'a, 'typeck, 'tcx> LivenessContext<'a, 'typeck, 'tcx> {
+    /// Computes the `MaybeInitializedPlaces` dataflow analysis if it hasn't been done already.
+    ///
+    /// In practice, the results of this dataflow analysis are rarely needed but can be expensive to
+    /// compute on big functions, so we compute them lazily as a fast path when:
+    /// - there are relevant live locals
+    /// - there are drop points for these relevant live locals.
+    ///
+    /// This happens as part of the drop-liveness computation: it's the only place checking for
+    /// maybe-initializedness of `MovePathIndex`es.
+    fn flow_inits(&mut self) -> &mut ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>> {
+        self.flow_inits.get_or_insert_with(|| {
+            let tcx = self.typeck.tcx();
+            let body = self.typeck.body;
+            // FIXME: reduce the `MaybeInitializedPlaces` domain to the useful `MovePath`s.
+            //
+            // This dataflow analysis computes maybe-initializedness of all move paths, which
+            // explains why it can be expensive on big functions. But this data is only used in
+            // drop-liveness. Therefore, most of the move paths computed here are ultimately unused,
+            // even if the results are computed lazily and "no relevant live locals with drop
+            // points" is the common case.
+            //
+            // So we only need the ones for 1) relevant live locals 2) that have drop points. That's
+            // a much, much smaller domain: in our benchmarks, when it's not zero (the most likely
+            // case), there are a few dozens compared to e.g. thousands or tens of thousands of
+            // locals and move paths.
+            let flow_inits = MaybeInitializedPlaces::new(tcx, body, self.move_data)
+                .iterate_to_fixpoint(tcx, body, Some("borrowck"))
+                .into_results_cursor(body);
+            flow_inits
+        })
+    }
+}
+
+impl<'tcx> LivenessContext<'_, '_, 'tcx> {
     fn body(&self) -> &Body<'tcx> {
         self.typeck.body
     }
+
     /// Returns `true` if the local variable (or some part of it) is initialized at the current
     /// cursor position. Callers should call one of the `seek` methods immediately before to point
     /// the cursor to the desired location.
-    fn initialized_at_curr_loc(&self, mpi: MovePathIndex) -> bool {
-        let state = self.flow_inits.get();
+    fn initialized_at_curr_loc(&mut self, mpi: MovePathIndex) -> bool {
+        let flow_inits = self.flow_inits();
+        let state = flow_inits.get();
         if state.contains(mpi) {
             return true;
         }
 
-        let move_paths = &self.flow_inits.analysis().move_data().move_paths;
+        let move_paths = &flow_inits.analysis().move_data().move_paths;
         move_paths[mpi].find_descendant(move_paths, |mpi| state.contains(mpi)).is_some()
     }
 
@@ -481,7 +516,8 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
     /// DROP of some local variable will have an effect -- note that
     /// drops, as they may unwind, are always terminators.
     fn initialized_at_terminator(&mut self, block: BasicBlock, mpi: MovePathIndex) -> bool {
-        self.flow_inits.seek_before_primary_effect(self.body().terminator_loc(block));
+        let terminator_location = self.body().terminator_loc(block);
+        self.flow_inits().seek_before_primary_effect(terminator_location);
         self.initialized_at_curr_loc(mpi)
     }
 
@@ -491,7 +527,8 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
     /// **Warning:** Does not account for the result of `Call`
     /// instructions.
     fn initialized_at_exit(&mut self, block: BasicBlock, mpi: MovePathIndex) -> bool {
-        self.flow_inits.seek_after_primary_effect(self.body().terminator_loc(block));
+        let terminator_location = self.body().terminator_loc(block);
+        self.flow_inits().seek_after_primary_effect(terminator_location);
         self.initialized_at_curr_loc(mpi)
     }
 
diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs
index 8c512257120..4f75dd7e992 100644
--- a/compiler/rustc_borrowck/src/type_check/mod.rs
+++ b/compiler/rustc_borrowck/src/type_check/mod.rs
@@ -30,8 +30,6 @@ use rustc_middle::ty::{
     TypeVisitableExt, UserArgs, UserTypeAnnotationIndex, fold_regions,
 };
 use rustc_middle::{bug, span_bug};
-use rustc_mir_dataflow::ResultsCursor;
-use rustc_mir_dataflow::impls::MaybeInitializedPlaces;
 use rustc_mir_dataflow::move_paths::MoveData;
 use rustc_mir_dataflow::points::DenseLocationMap;
 use rustc_span::def_id::CRATE_DEF_ID;
@@ -97,10 +95,9 @@ mod relate_tys;
 /// - `location_table` -- for datalog polonius, the map between `Location`s and `RichLocation`s
 /// - `borrow_set` -- information about borrows occurring in `body`
 /// - `polonius_facts` -- when using Polonius, this is the generated set of Polonius facts
-/// - `flow_inits` -- results of a maybe-init dataflow analysis
 /// - `move_data` -- move-data constructed when performing the maybe-init dataflow analysis
 /// - `location_map` -- map between MIR `Location` and `PointIndex`
-pub(crate) fn type_check<'a, 'tcx>(
+pub(crate) fn type_check<'tcx>(
     root_cx: &mut BorrowCheckRootCtxt<'tcx>,
     infcx: &BorrowckInferCtxt<'tcx>,
     body: &Body<'tcx>,
@@ -109,7 +106,6 @@ pub(crate) fn type_check<'a, 'tcx>(
     location_table: &PoloniusLocationTable,
     borrow_set: &BorrowSet<'tcx>,
     polonius_facts: &mut Option<PoloniusFacts>,
-    flow_inits: ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>>,
     move_data: &MoveData<'tcx>,
     location_map: Rc<DenseLocationMap>,
 ) -> MirTypeckResults<'tcx> {
@@ -167,7 +163,7 @@ pub(crate) fn type_check<'a, 'tcx>(
     typeck.equate_inputs_and_outputs(&normalized_inputs_and_output);
     typeck.check_signature_annotation();
 
-    liveness::generate(&mut typeck, &location_map, flow_inits, move_data);
+    liveness::generate(&mut typeck, &location_map, move_data);
 
     let opaque_type_values =
         opaque_types::take_opaques_and_register_member_constraints(&mut typeck);
@@ -474,17 +470,18 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             let projected_ty = curr_projected_ty.projection_ty_core(
                 tcx,
                 proj,
-                |this, field, ()| {
-                    let ty = this.field_ty(tcx, field);
-                    self.structurally_resolve(ty, locations)
-                },
-                |_, _| unreachable!(),
+                |ty| self.structurally_resolve(ty, locations),
+                |ty, variant_index, field, ()| PlaceTy::field_ty(tcx, ty, variant_index, field),
+                |_| unreachable!(),
             );
             curr_projected_ty = projected_ty;
         }
         trace!(?curr_projected_ty);
 
-        let ty = curr_projected_ty.ty;
+        // Need to renormalize `a` as typecheck may have failed to normalize
+        // higher-ranked aliases if normalization was ambiguous due to inference.
+        let a = self.normalize(a, locations);
+        let ty = self.normalize(curr_projected_ty.ty, locations);
         self.relate_types(ty, v.xform(ty::Contravariant), a, locations, category)?;
 
         Ok(())
@@ -691,7 +688,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                 if !self.unsized_feature_enabled() {
                     let trait_ref = ty::TraitRef::new(
                         tcx,
-                        tcx.require_lang_item(LangItem::Sized, Some(self.last_span)),
+                        tcx.require_lang_item(LangItem::Sized, self.last_span),
                         [place_ty],
                     );
                     self.prove_trait_ref(
@@ -1013,7 +1010,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                             let ty = place.ty(self.body, tcx).ty;
                             let trait_ref = ty::TraitRef::new(
                                 tcx,
-                                tcx.require_lang_item(LangItem::Copy, Some(span)),
+                                tcx.require_lang_item(LangItem::Copy, span),
                                 [ty],
                             );
 
@@ -1028,11 +1025,8 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
             }
 
             &Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf, ty) => {
-                let trait_ref = ty::TraitRef::new(
-                    tcx,
-                    tcx.require_lang_item(LangItem::Sized, Some(span)),
-                    [ty],
-                );
+                let trait_ref =
+                    ty::TraitRef::new(tcx, tcx.require_lang_item(LangItem::Sized, span), [ty]);
 
                 self.prove_trait_ref(
                     trait_ref,
@@ -1044,11 +1038,8 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
             &Rvalue::NullaryOp(NullOp::UbChecks, _) => {}
 
             Rvalue::ShallowInitBox(_operand, ty) => {
-                let trait_ref = ty::TraitRef::new(
-                    tcx,
-                    tcx.require_lang_item(LangItem::Sized, Some(span)),
-                    [*ty],
-                );
+                let trait_ref =
+                    ty::TraitRef::new(tcx, tcx.require_lang_item(LangItem::Sized, span), [*ty]);
 
                 self.prove_trait_ref(
                     trait_ref,
@@ -1225,7 +1216,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                         let &ty = ty;
                         let trait_ref = ty::TraitRef::new(
                             tcx,
-                            tcx.require_lang_item(LangItem::CoerceUnsized, Some(span)),
+                            tcx.require_lang_item(LangItem::CoerceUnsized, span),
                             [op.ty(self.body, tcx), ty],
                         );
 
@@ -1814,7 +1805,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
         if let PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy) = context {
             let trait_ref = ty::TraitRef::new(
                 tcx,
-                tcx.require_lang_item(LangItem::Copy, Some(self.last_span)),
+                tcx.require_lang_item(LangItem::Copy, self.last_span),
                 [place_ty.ty],
             );
 
@@ -1852,7 +1843,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
             | ProjectionElem::Downcast(..) => {}
             ProjectionElem::Field(field, fty) => {
                 let fty = self.normalize(fty, location);
-                let ty = base_ty.field_ty(tcx, field);
+                let ty = PlaceTy::field_ty(tcx, base_ty.ty, base_ty.variant_index, field);
                 let ty = self.normalize(ty, location);
                 debug!(?fty, ?ty);
 
diff --git a/compiler/rustc_borrowck/src/type_check/opaque_types.rs b/compiler/rustc_borrowck/src/type_check/opaque_types.rs
index 341c50c37f6..5a422483eef 100644
--- a/compiler/rustc_borrowck/src/type_check/opaque_types.rs
+++ b/compiler/rustc_borrowck/src/type_check/opaque_types.rs
@@ -221,7 +221,7 @@ fn register_member_constraints<'tcx>(
         .iter()
         .enumerate()
         .filter(|(i, _)| variances[*i] == ty::Invariant)
-        .filter_map(|(_, arg)| match arg.unpack() {
+        .filter_map(|(_, arg)| match arg.kind() {
             GenericArgKind::Lifetime(r) => Some(typeck.to_region_vid(r)),
             GenericArgKind::Type(_) | GenericArgKind::Const(_) => None,
         })
diff --git a/compiler/rustc_borrowck/src/universal_regions.rs b/compiler/rustc_borrowck/src/universal_regions.rs
index c11e14d214c..846299711be 100644
--- a/compiler/rustc_borrowck/src/universal_regions.rs
+++ b/compiler/rustc_borrowck/src/universal_regions.rs
@@ -544,10 +544,10 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
         // (as it's created inside the body itself, not passed in from outside).
         if let DefiningTy::FnDef(def_id, _) = defining_ty {
             if self.infcx.tcx.fn_sig(def_id).skip_binder().c_variadic() {
-                let va_list_did = self.infcx.tcx.require_lang_item(
-                    LangItem::VaList,
-                    Some(self.infcx.tcx.def_span(self.mir_def)),
-                );
+                let va_list_did = self
+                    .infcx
+                    .tcx
+                    .require_lang_item(LangItem::VaList, self.infcx.tcx.def_span(self.mir_def));
 
                 let reg_vid = self
                     .infcx
diff --git a/compiler/rustc_builtin_macros/Cargo.toml b/compiler/rustc_builtin_macros/Cargo.toml
index 5c1ae90f729..4c1264c6f1c 100644
--- a/compiler/rustc_builtin_macros/Cargo.toml
+++ b/compiler/rustc_builtin_macros/Cargo.toml
@@ -24,6 +24,9 @@ rustc_lint_defs = { path = "../rustc_lint_defs" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_parse = { path = "../rustc_parse" }
 rustc_parse_format = { path = "../rustc_parse_format" }
+# We must use the proc_macro version that we will compile proc-macros against,
+# not the one from our own sysroot.
+rustc_proc_macro = { path = "../rustc_proc_macro" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
diff --git a/compiler/rustc_builtin_macros/messages.ftl b/compiler/rustc_builtin_macros/messages.ftl
index 73be954cefd..d32e6f1558e 100644
--- a/compiler/rustc_builtin_macros/messages.ftl
+++ b/compiler/rustc_builtin_macros/messages.ftl
@@ -1,6 +1,11 @@
 builtin_macros_alloc_error_must_be_fn = alloc_error_handler must be a function
 builtin_macros_alloc_must_statics = allocators must be statics
 
+builtin_macros_asm_attribute_not_supported =
+    this attribute is not supported on assembly
+builtin_macros_asm_cfg =
+    the `#[cfg(/* ... */)]` and `#[cfg_attr(/* ... */)]` attributes on assembly are unstable
+
 builtin_macros_asm_clobber_abi = clobber_abi
 builtin_macros_asm_clobber_no_reg = asm with `clobber_abi` must specify explicit registers for outputs
 builtin_macros_asm_clobber_outputs = generic outputs
@@ -9,17 +14,6 @@ builtin_macros_asm_duplicate_arg = duplicate argument named `{$name}`
     .label = previously here
     .arg = duplicate argument
 
-builtin_macros_asm_expected_comma = expected token: `,`
-    .label = expected `,`
-
-builtin_macros_asm_expected_other = expected operand, {$is_inline_asm ->
-    [false] options
-    *[true] clobber_abi, options
-    }, or additional template string
-
-builtin_macros_asm_expected_string_literal = expected string literal
-    .label = not a string literal
-
 builtin_macros_asm_explicit_register_name = explicit register arguments cannot have names
 
 builtin_macros_asm_mayunwind = asm labels are not allowed with the `may_unwind` option
@@ -45,17 +39,8 @@ builtin_macros_asm_pure_combine = the `pure` option must be combined with either
 
 builtin_macros_asm_pure_no_output = asm with the `pure` option must have at least one output
 
-builtin_macros_asm_requires_template = requires at least a template string argument
-
-builtin_macros_asm_sym_no_path = expected a path for argument to `sym`
-
-builtin_macros_asm_underscore_input = _ cannot be used for input operands
-
 builtin_macros_asm_unsupported_clobber_abi = `clobber_abi` cannot be used with `{$macro_name}!`
 
-builtin_macros_asm_unsupported_operand = the `{$symbol}` operand cannot be used with `{$macro_name}!`
-    .label = the `{$symbol}` operand is not meaningful for global-scoped inline assembly, remove it
-
 builtin_macros_asm_unsupported_option = the `{$symbol}` option cannot be used with `{$macro_name}!`
     .label = the `{$symbol}` option is not meaningful for global-scoped inline assembly
     .suggestion = remove this option
@@ -71,7 +56,6 @@ builtin_macros_assert_requires_expression = macro requires an expression as an a
 
 builtin_macros_autodiff = autodiff must be applied to function
 builtin_macros_autodiff_missing_config = autodiff requires at least a name and mode
-builtin_macros_autodiff_mode = unknown Mode: `{$mode}`. Use `Forward` or `Reverse`
 builtin_macros_autodiff_mode_activity = {$act} can not be used in {$mode} Mode
 builtin_macros_autodiff_not_build = this rustc version does not support autodiff
 builtin_macros_autodiff_number_activities = expected {$expected} activities, but found {$found}
@@ -162,7 +146,10 @@ builtin_macros_expected_comma_in_list = expected token: `,`
 
 builtin_macros_expected_one_cfg_pattern = expected 1 cfg-pattern
 
-builtin_macros_expected_register_class_or_explicit_register = expected register class or explicit register
+builtin_macros_expected_other = expected operand, {$is_inline_asm ->
+    [false] options
+    *[true] clobber_abi, options
+    }, or additional template string
 
 builtin_macros_export_macro_rules = cannot export macro_rules! macros from a `proc-macro` crate type currently
 
@@ -255,8 +242,6 @@ builtin_macros_no_default_variant = `#[derive(Default)]` on enum with no `#[defa
     .label = this enum needs a unit variant marked with `#[default]`
     .suggestion = make this unit variant default by placing `#[default]` on it
 
-builtin_macros_non_abi = at least one abi must be provided as an argument to `clobber_abi`
-
 builtin_macros_non_exhaustive_default = default variant must be exhaustive
     .label = declared `#[non_exhaustive]` here
     .help = consider a manual implementation of `Default`
diff --git a/compiler/rustc_builtin_macros/src/asm.rs b/compiler/rustc_builtin_macros/src/asm.rs
index 62ee71fecc2..1fb99817222 100644
--- a/compiler/rustc_builtin_macros/src/asm.rs
+++ b/compiler/rustc_builtin_macros/src/asm.rs
@@ -1,4 +1,3 @@
-use ast::token::IdentIsRaw;
 use lint::BuiltinLintDiag;
 use rustc_ast::ptr::P;
 use rustc_ast::tokenstream::TokenStream;
@@ -7,39 +6,16 @@ use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
 use rustc_errors::PResult;
 use rustc_expand::base::*;
 use rustc_index::bit_set::GrowableBitSet;
-use rustc_parse::exp;
-use rustc_parse::parser::{ExpKeywordPair, Parser};
+use rustc_parse::parser::asm::*;
 use rustc_session::lint;
-use rustc_span::{ErrorGuaranteed, InnerSpan, Span, Symbol, kw};
+use rustc_session::parse::feature_err;
+use rustc_span::{ErrorGuaranteed, InnerSpan, Span, Symbol, sym};
 use rustc_target::asm::InlineAsmArch;
 use smallvec::smallvec;
 use {rustc_ast as ast, rustc_parse_format as parse};
 
-use crate::errors;
 use crate::util::{ExprToSpannedString, expr_to_spanned_string};
-
-/// An argument to one of the `asm!` macros. The argument is syntactically valid, but is otherwise
-/// not validated at all.
-pub struct AsmArg {
-    pub kind: AsmArgKind,
-    pub span: Span,
-}
-
-pub enum AsmArgKind {
-    Template(P<ast::Expr>),
-    Operand(Option<Symbol>, ast::InlineAsmOperand),
-    Options(Vec<AsmOption>),
-    ClobberAbi(Vec<(Symbol, Span)>),
-}
-
-pub struct AsmOption {
-    pub symbol: Symbol,
-    pub span: Span,
-    // A bitset, with only the bit for this option's symbol set.
-    pub options: ast::InlineAsmOptions,
-    // Used when suggesting to remove an option.
-    pub span_with_comma: Span,
-}
+use crate::{errors, fluent_generated as fluent};
 
 /// Validated assembly arguments, ready for macro expansion.
 struct ValidatedAsmArgs {
@@ -52,215 +28,6 @@ struct ValidatedAsmArgs {
     pub options_spans: Vec<Span>,
 }
 
-/// Used for better error messages when operand types are used that are not
-/// supported by the current macro (e.g. `in` or `out` for `global_asm!`)
-///
-/// returns
-///
-/// - `Ok(true)` if the current token matches the keyword, and was expected
-/// - `Ok(false)` if the current token does not match the keyword
-/// - `Err(_)` if the current token matches the keyword, but was not expected
-fn eat_operand_keyword<'a>(
-    p: &mut Parser<'a>,
-    exp: ExpKeywordPair,
-    asm_macro: AsmMacro,
-) -> PResult<'a, bool> {
-    if matches!(asm_macro, AsmMacro::Asm) {
-        Ok(p.eat_keyword(exp))
-    } else {
-        let span = p.token.span;
-        if p.eat_keyword_noexpect(exp.kw) {
-            // in gets printed as `r#in` otherwise
-            let symbol = if exp.kw == kw::In { "in" } else { exp.kw.as_str() };
-            Err(p.dcx().create_err(errors::AsmUnsupportedOperand {
-                span,
-                symbol,
-                macro_name: asm_macro.macro_name(),
-            }))
-        } else {
-            Ok(false)
-        }
-    }
-}
-
-fn parse_asm_operand<'a>(
-    p: &mut Parser<'a>,
-    asm_macro: AsmMacro,
-) -> PResult<'a, Option<ast::InlineAsmOperand>> {
-    let dcx = p.dcx();
-
-    Ok(Some(if eat_operand_keyword(p, exp!(In), asm_macro)? {
-        let reg = parse_reg(p)?;
-        if p.eat_keyword(exp!(Underscore)) {
-            let err = dcx.create_err(errors::AsmUnderscoreInput { span: p.token.span });
-            return Err(err);
-        }
-        let expr = p.parse_expr()?;
-        ast::InlineAsmOperand::In { reg, expr }
-    } else if eat_operand_keyword(p, exp!(Out), asm_macro)? {
-        let reg = parse_reg(p)?;
-        let expr = if p.eat_keyword(exp!(Underscore)) { None } else { Some(p.parse_expr()?) };
-        ast::InlineAsmOperand::Out { reg, expr, late: false }
-    } else if eat_operand_keyword(p, exp!(Lateout), asm_macro)? {
-        let reg = parse_reg(p)?;
-        let expr = if p.eat_keyword(exp!(Underscore)) { None } else { Some(p.parse_expr()?) };
-        ast::InlineAsmOperand::Out { reg, expr, late: true }
-    } else if eat_operand_keyword(p, exp!(Inout), asm_macro)? {
-        let reg = parse_reg(p)?;
-        if p.eat_keyword(exp!(Underscore)) {
-            let err = dcx.create_err(errors::AsmUnderscoreInput { span: p.token.span });
-            return Err(err);
-        }
-        let expr = p.parse_expr()?;
-        if p.eat(exp!(FatArrow)) {
-            let out_expr =
-                if p.eat_keyword(exp!(Underscore)) { None } else { Some(p.parse_expr()?) };
-            ast::InlineAsmOperand::SplitInOut { reg, in_expr: expr, out_expr, late: false }
-        } else {
-            ast::InlineAsmOperand::InOut { reg, expr, late: false }
-        }
-    } else if eat_operand_keyword(p, exp!(Inlateout), asm_macro)? {
-        let reg = parse_reg(p)?;
-        if p.eat_keyword(exp!(Underscore)) {
-            let err = dcx.create_err(errors::AsmUnderscoreInput { span: p.token.span });
-            return Err(err);
-        }
-        let expr = p.parse_expr()?;
-        if p.eat(exp!(FatArrow)) {
-            let out_expr =
-                if p.eat_keyword(exp!(Underscore)) { None } else { Some(p.parse_expr()?) };
-            ast::InlineAsmOperand::SplitInOut { reg, in_expr: expr, out_expr, late: true }
-        } else {
-            ast::InlineAsmOperand::InOut { reg, expr, late: true }
-        }
-    } else if eat_operand_keyword(p, exp!(Label), asm_macro)? {
-        let block = p.parse_block()?;
-        ast::InlineAsmOperand::Label { block }
-    } else if p.eat_keyword(exp!(Const)) {
-        let anon_const = p.parse_expr_anon_const()?;
-        ast::InlineAsmOperand::Const { anon_const }
-    } else if p.eat_keyword(exp!(Sym)) {
-        let expr = p.parse_expr()?;
-        let ast::ExprKind::Path(qself, path) = &expr.kind else {
-            let err = dcx.create_err(errors::AsmSymNoPath { span: expr.span });
-            return Err(err);
-        };
-        let sym =
-            ast::InlineAsmSym { id: ast::DUMMY_NODE_ID, qself: qself.clone(), path: path.clone() };
-        ast::InlineAsmOperand::Sym { sym }
-    } else {
-        return Ok(None);
-    }))
-}
-
-// Public for rustfmt.
-pub fn parse_asm_args<'a>(
-    p: &mut Parser<'a>,
-    sp: Span,
-    asm_macro: AsmMacro,
-) -> PResult<'a, Vec<AsmArg>> {
-    let dcx = p.dcx();
-
-    if p.token == token::Eof {
-        return Err(dcx.create_err(errors::AsmRequiresTemplate { span: sp }));
-    }
-
-    let mut args = Vec::new();
-
-    let first_template = p.parse_expr()?;
-    args.push(AsmArg { span: first_template.span, kind: AsmArgKind::Template(first_template) });
-
-    let mut allow_templates = true;
-
-    while p.token != token::Eof {
-        if !p.eat(exp!(Comma)) {
-            if allow_templates {
-                // After a template string, we always expect *only* a comma...
-                return Err(dcx.create_err(errors::AsmExpectedComma { span: p.token.span }));
-            } else {
-                // ...after that delegate to `expect` to also include the other expected tokens.
-                return Err(p.expect(exp!(Comma)).err().unwrap());
-            }
-        }
-
-        // Accept trailing commas.
-        if p.token == token::Eof {
-            break;
-        }
-
-        let span_start = p.token.span;
-
-        // Parse `clobber_abi`.
-        if p.eat_keyword(exp!(ClobberAbi)) {
-            allow_templates = false;
-
-            args.push(AsmArg {
-                kind: AsmArgKind::ClobberAbi(parse_clobber_abi(p)?),
-                span: span_start.to(p.prev_token.span),
-            });
-
-            continue;
-        }
-
-        // Parse `options`.
-        if p.eat_keyword(exp!(Options)) {
-            allow_templates = false;
-
-            args.push(AsmArg {
-                kind: AsmArgKind::Options(parse_options(p, asm_macro)?),
-                span: span_start.to(p.prev_token.span),
-            });
-
-            continue;
-        }
-
-        // Parse operand names.
-        let name = if p.token.is_ident() && p.look_ahead(1, |t| *t == token::Eq) {
-            let (ident, _) = p.token.ident().unwrap();
-            p.bump();
-            p.expect(exp!(Eq))?;
-            allow_templates = false;
-            Some(ident.name)
-        } else {
-            None
-        };
-
-        if let Some(op) = parse_asm_operand(p, asm_macro)? {
-            allow_templates = false;
-
-            args.push(AsmArg {
-                span: span_start.to(p.prev_token.span),
-                kind: AsmArgKind::Operand(name, op),
-            });
-        } else if allow_templates {
-            let template = p.parse_expr()?;
-            // If it can't possibly expand to a string, provide diagnostics here to include other
-            // things it could have been.
-            match template.kind {
-                ast::ExprKind::Lit(token_lit)
-                    if matches!(
-                        token_lit.kind,
-                        token::LitKind::Str | token::LitKind::StrRaw(_)
-                    ) => {}
-                ast::ExprKind::MacCall(..) => {}
-                _ => {
-                    let err = dcx.create_err(errors::AsmExpectedOther {
-                        span: template.span,
-                        is_inline_asm: matches!(asm_macro, AsmMacro::Asm),
-                    });
-                    return Err(err);
-                }
-            }
-
-            args.push(AsmArg { span: template.span, kind: AsmArgKind::Template(template) });
-        } else {
-            p.unexpected_any()?
-        }
-    }
-
-    Ok(args)
-}
-
 fn parse_args<'a>(
     ecx: &ExtCtxt<'a>,
     sp: Span,
@@ -278,6 +45,13 @@ fn validate_asm_args<'a>(
 ) -> PResult<'a, ValidatedAsmArgs> {
     let dcx = ecx.dcx();
 
+    let strip_unconfigured = rustc_expand::config::StripUnconfigured {
+        sess: ecx.sess,
+        features: Some(ecx.ecfg.features),
+        config_tokens: false,
+        lint_node_id: ecx.current_expansion.lint_node_id,
+    };
+
     let mut validated = ValidatedAsmArgs {
         templates: vec![],
         operands: vec![],
@@ -291,6 +65,26 @@ fn validate_asm_args<'a>(
     let mut allow_templates = true;
 
     for arg in args {
+        for attr in arg.attributes.0.iter() {
+            match attr.name() {
+                Some(sym::cfg | sym::cfg_attr) => {
+                    if !ecx.ecfg.features.asm_cfg() {
+                        let span = attr.span();
+                        feature_err(ecx.sess, sym::asm_cfg, span, fluent::builtin_macros_asm_cfg)
+                            .emit();
+                    }
+                }
+                _ => {
+                    ecx.dcx().emit_err(errors::AsmAttributeNotSupported { span: attr.span() });
+                }
+            }
+        }
+
+        // Skip arguments that are configured out.
+        if ecx.ecfg.features.asm_cfg() && strip_unconfigured.configure(arg.attributes).is_none() {
+            continue;
+        }
+
         match arg.kind {
             AsmArgKind::Template(template) => {
                 // The error for the first template is delayed.
@@ -479,103 +273,6 @@ fn validate_asm_args<'a>(
     Ok(validated)
 }
 
-fn parse_options<'a>(p: &mut Parser<'a>, asm_macro: AsmMacro) -> PResult<'a, Vec<AsmOption>> {
-    p.expect(exp!(OpenParen))?;
-
-    let mut asm_options = Vec::new();
-
-    while !p.eat(exp!(CloseParen)) {
-        const OPTIONS: [(ExpKeywordPair, ast::InlineAsmOptions); ast::InlineAsmOptions::COUNT] = [
-            (exp!(Pure), ast::InlineAsmOptions::PURE),
-            (exp!(Nomem), ast::InlineAsmOptions::NOMEM),
-            (exp!(Readonly), ast::InlineAsmOptions::READONLY),
-            (exp!(PreservesFlags), ast::InlineAsmOptions::PRESERVES_FLAGS),
-            (exp!(Noreturn), ast::InlineAsmOptions::NORETURN),
-            (exp!(Nostack), ast::InlineAsmOptions::NOSTACK),
-            (exp!(MayUnwind), ast::InlineAsmOptions::MAY_UNWIND),
-            (exp!(AttSyntax), ast::InlineAsmOptions::ATT_SYNTAX),
-            (exp!(Raw), ast::InlineAsmOptions::RAW),
-        ];
-
-        'blk: {
-            for (exp, options) in OPTIONS {
-                // Gives a more accurate list of expected next tokens.
-                let kw_matched = if asm_macro.is_supported_option(options) {
-                    p.eat_keyword(exp)
-                } else {
-                    p.eat_keyword_noexpect(exp.kw)
-                };
-
-                if kw_matched {
-                    let span = p.prev_token.span;
-                    let span_with_comma =
-                        if p.token == token::Comma { span.to(p.token.span) } else { span };
-
-                    asm_options.push(AsmOption { symbol: exp.kw, span, options, span_with_comma });
-                    break 'blk;
-                }
-            }
-
-            return p.unexpected_any();
-        }
-
-        // Allow trailing commas.
-        if p.eat(exp!(CloseParen)) {
-            break;
-        }
-        p.expect(exp!(Comma))?;
-    }
-
-    Ok(asm_options)
-}
-
-fn parse_clobber_abi<'a>(p: &mut Parser<'a>) -> PResult<'a, Vec<(Symbol, Span)>> {
-    p.expect(exp!(OpenParen))?;
-
-    if p.eat(exp!(CloseParen)) {
-        return Err(p.dcx().create_err(errors::NonABI { span: p.token.span }));
-    }
-
-    let mut new_abis = Vec::new();
-    while !p.eat(exp!(CloseParen)) {
-        match p.parse_str_lit() {
-            Ok(str_lit) => {
-                new_abis.push((str_lit.symbol_unescaped, str_lit.span));
-            }
-            Err(opt_lit) => {
-                let span = opt_lit.map_or(p.token.span, |lit| lit.span);
-                return Err(p.dcx().create_err(errors::AsmExpectedStringLiteral { span }));
-            }
-        };
-
-        // Allow trailing commas
-        if p.eat(exp!(CloseParen)) {
-            break;
-        }
-        p.expect(exp!(Comma))?;
-    }
-
-    Ok(new_abis)
-}
-
-fn parse_reg<'a>(p: &mut Parser<'a>) -> PResult<'a, ast::InlineAsmRegOrRegClass> {
-    p.expect(exp!(OpenParen))?;
-    let result = match p.token.uninterpolate().kind {
-        token::Ident(name, IdentIsRaw::No) => ast::InlineAsmRegOrRegClass::RegClass(name),
-        token::Literal(token::Lit { kind: token::LitKind::Str, symbol, suffix: _ }) => {
-            ast::InlineAsmRegOrRegClass::Reg(symbol)
-        }
-        _ => {
-            return Err(p.dcx().create_err(errors::ExpectedRegisterClassOrExplicitRegister {
-                span: p.token.span,
-            }));
-        }
-    };
-    p.bump();
-    p.expect(exp!(CloseParen))?;
-    Ok(result)
-}
-
 fn expand_preparsed_asm(
     ecx: &mut ExtCtxt<'_>,
     asm_macro: AsmMacro,
diff --git a/compiler/rustc_builtin_macros/src/autodiff.rs b/compiler/rustc_builtin_macros/src/autodiff.rs
index 1ff4fc6aaab..dc3bb8ab52a 100644
--- a/compiler/rustc_builtin_macros/src/autodiff.rs
+++ b/compiler/rustc_builtin_macros/src/autodiff.rs
@@ -86,27 +86,23 @@ mod llvm_enzyme {
         ecx: &mut ExtCtxt<'_>,
         meta_item: &ThinVec<MetaItemInner>,
         has_ret: bool,
+        mode: DiffMode,
     ) -> AutoDiffAttrs {
         let dcx = ecx.sess.dcx();
-        let mode = name(&meta_item[1]);
-        let Ok(mode) = DiffMode::from_str(&mode) else {
-            dcx.emit_err(errors::AutoDiffInvalidMode { span: meta_item[1].span(), mode });
-            return AutoDiffAttrs::error();
-        };
 
         // Now we check, whether the user wants autodiff in batch/vector mode, or scalar mode.
         // If he doesn't specify an integer (=width), we default to scalar mode, thus width=1.
-        let mut first_activity = 2;
+        let mut first_activity = 1;
 
-        let width = if let [_, _, x, ..] = &meta_item[..]
+        let width = if let [_, x, ..] = &meta_item[..]
             && let Some(x) = width(x)
         {
-            first_activity = 3;
+            first_activity = 2;
             match x.try_into() {
                 Ok(x) => x,
                 Err(_) => {
                     dcx.emit_err(errors::AutoDiffInvalidWidth {
-                        span: meta_item[2].span(),
+                        span: meta_item[1].span(),
                         width: x,
                     });
                     return AutoDiffAttrs::error();
@@ -165,6 +161,24 @@ mod llvm_enzyme {
         ts.push(TokenTree::Token(comma.clone(), Spacing::Alone));
     }
 
+    pub(crate) fn expand_forward(
+        ecx: &mut ExtCtxt<'_>,
+        expand_span: Span,
+        meta_item: &ast::MetaItem,
+        item: Annotatable,
+    ) -> Vec<Annotatable> {
+        expand_with_mode(ecx, expand_span, meta_item, item, DiffMode::Forward)
+    }
+
+    pub(crate) fn expand_reverse(
+        ecx: &mut ExtCtxt<'_>,
+        expand_span: Span,
+        meta_item: &ast::MetaItem,
+        item: Annotatable,
+    ) -> Vec<Annotatable> {
+        expand_with_mode(ecx, expand_span, meta_item, item, DiffMode::Reverse)
+    }
+
     /// We expand the autodiff macro to generate a new placeholder function which passes
     /// type-checking and can be called by users. The function body of the placeholder function will
     /// later be replaced on LLVM-IR level, so the design of the body is less important and for now
@@ -198,11 +212,12 @@ mod llvm_enzyme {
     /// ```
     /// FIXME(ZuseZ4): Once autodiff is enabled by default, make this a doc comment which is checked
     /// in CI.
-    pub(crate) fn expand(
+    pub(crate) fn expand_with_mode(
         ecx: &mut ExtCtxt<'_>,
         expand_span: Span,
         meta_item: &ast::MetaItem,
         mut item: Annotatable,
+        mode: DiffMode,
     ) -> Vec<Annotatable> {
         if cfg!(not(llvm_enzyme)) {
             ecx.sess.dcx().emit_err(errors::AutoDiffSupportNotBuild { span: meta_item.span });
@@ -245,29 +260,41 @@ mod llvm_enzyme {
         // create TokenStream from vec elemtents:
         // meta_item doesn't have a .tokens field
         let mut ts: Vec<TokenTree> = vec![];
-        if meta_item_vec.len() < 2 {
-            // At the bare minimum, we need a fnc name and a mode, even for a dummy function with no
-            // input and output args.
+        if meta_item_vec.len() < 1 {
+            // At the bare minimum, we need a fnc name.
             dcx.emit_err(errors::AutoDiffMissingConfig { span: item.span() });
             return vec![item];
         }
 
-        meta_item_inner_to_ts(&meta_item_vec[1], &mut ts);
+        let mode_symbol = match mode {
+            DiffMode::Forward => sym::Forward,
+            DiffMode::Reverse => sym::Reverse,
+            _ => unreachable!("Unsupported mode: {:?}", mode),
+        };
+
+        // Insert mode token
+        let mode_token = Token::new(TokenKind::Ident(mode_symbol, false.into()), Span::default());
+        ts.insert(0, TokenTree::Token(mode_token, Spacing::Joint));
+        ts.insert(
+            1,
+            TokenTree::Token(Token::new(TokenKind::Comma, Span::default()), Spacing::Alone),
+        );
 
         // Now, if the user gave a width (vector aka batch-mode ad), then we copy it.
         // If it is not given, we default to 1 (scalar mode).
         let start_position;
         let kind: LitKind = LitKind::Integer;
         let symbol;
-        if meta_item_vec.len() >= 3
-            && let Some(width) = width(&meta_item_vec[2])
+        if meta_item_vec.len() >= 2
+            && let Some(width) = width(&meta_item_vec[1])
         {
-            start_position = 3;
+            start_position = 2;
             symbol = Symbol::intern(&width.to_string());
         } else {
-            start_position = 2;
+            start_position = 1;
             symbol = sym::integer(1);
         }
+
         let l: Lit = Lit { kind, symbol, suffix: None };
         let t = Token::new(TokenKind::Literal(l), Span::default());
         let comma = Token::new(TokenKind::Comma, Span::default());
@@ -289,7 +316,7 @@ mod llvm_enzyme {
         ts.pop();
         let ts: TokenStream = TokenStream::from_iter(ts);
 
-        let x: AutoDiffAttrs = from_ast(ecx, &meta_item_vec, has_ret);
+        let x: AutoDiffAttrs = from_ast(ecx, &meta_item_vec, has_ret, mode);
         if !x.is_active() {
             // We encountered an error, so we return the original item.
             // This allows us to potentially parse other attributes.
@@ -1017,4 +1044,4 @@ mod llvm_enzyme {
     }
 }
 
-pub(crate) use llvm_enzyme::expand;
+pub(crate) use llvm_enzyme::{expand_forward, expand_reverse};
diff --git a/compiler/rustc_builtin_macros/src/deriving/clone.rs b/compiler/rustc_builtin_macros/src/deriving/clone.rs
index 44cf215c662..69f8c273797 100644
--- a/compiler/rustc_builtin_macros/src/deriving/clone.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/clone.rs
@@ -34,8 +34,8 @@ pub(crate) fn expand_deriving_clone(
     let is_simple;
     match item {
         Annotatable::Item(annitem) => match &annitem.kind {
-            ItemKind::Struct(_, _, Generics { params, .. })
-            | ItemKind::Enum(_, _, Generics { params, .. }) => {
+            ItemKind::Struct(_, Generics { params, .. }, _)
+            | ItemKind::Enum(_, Generics { params, .. }, _) => {
                 let container_id = cx.current_expansion.id.expn_data().parent.expect_local();
                 let has_derive_copy = cx.resolver.has_derive_copy(container_id);
                 if has_derive_copy
diff --git a/compiler/rustc_builtin_macros/src/deriving/cmp/partial_eq.rs b/compiler/rustc_builtin_macros/src/deriving/cmp/partial_eq.rs
index 4b93b3414c7..b1d950b8d89 100644
--- a/compiler/rustc_builtin_macros/src/deriving/cmp/partial_eq.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/cmp/partial_eq.rs
@@ -8,6 +8,8 @@ use crate::deriving::generic::ty::*;
 use crate::deriving::generic::*;
 use crate::deriving::{path_local, path_std};
 
+/// Expands a `#[derive(PartialEq)]` attribute into an implementation for the
+/// target item.
 pub(crate) fn expand_deriving_partial_eq(
     cx: &ExtCtxt<'_>,
     span: Span,
@@ -16,62 +18,6 @@ pub(crate) fn expand_deriving_partial_eq(
     push: &mut dyn FnMut(Annotatable),
     is_const: bool,
 ) {
-    fn cs_eq(cx: &ExtCtxt<'_>, span: Span, substr: &Substructure<'_>) -> BlockOrExpr {
-        let base = true;
-        let expr = cs_fold(
-            true, // use foldl
-            cx,
-            span,
-            substr,
-            |cx, fold| match fold {
-                CsFold::Single(field) => {
-                    let [other_expr] = &field.other_selflike_exprs[..] else {
-                        cx.dcx()
-                            .span_bug(field.span, "not exactly 2 arguments in `derive(PartialEq)`");
-                    };
-
-                    // We received arguments of type `&T`. Convert them to type `T` by stripping
-                    // any leading `&`. This isn't necessary for type checking, but
-                    // it results in better error messages if something goes wrong.
-                    //
-                    // Note: for arguments that look like `&{ x }`, which occur with packed
-                    // structs, this would cause expressions like `{ self.x } == { other.x }`,
-                    // which isn't valid Rust syntax. This wouldn't break compilation because these
-                    // AST nodes are constructed within the compiler. But it would mean that code
-                    // printed by `-Zunpretty=expanded` (or `cargo expand`) would have invalid
-                    // syntax, which would be suboptimal. So we wrap these in parens, giving
-                    // `({ self.x }) == ({ other.x })`, which is valid syntax.
-                    let convert = |expr: &P<Expr>| {
-                        if let ExprKind::AddrOf(BorrowKind::Ref, Mutability::Not, inner) =
-                            &expr.kind
-                        {
-                            if let ExprKind::Block(..) = &inner.kind {
-                                // `&{ x }` form: remove the `&`, add parens.
-                                cx.expr_paren(field.span, inner.clone())
-                            } else {
-                                // `&x` form: remove the `&`.
-                                inner.clone()
-                            }
-                        } else {
-                            expr.clone()
-                        }
-                    };
-                    cx.expr_binary(
-                        field.span,
-                        BinOpKind::Eq,
-                        convert(&field.self_expr),
-                        convert(other_expr),
-                    )
-                }
-                CsFold::Combine(span, expr1, expr2) => {
-                    cx.expr_binary(span, BinOpKind::And, expr1, expr2)
-                }
-                CsFold::Fieldless => cx.expr_bool(span, base),
-            },
-        );
-        BlockOrExpr::new_expr(expr)
-    }
-
     let structural_trait_def = TraitDef {
         span,
         path: path_std!(marker::StructuralPartialEq),
@@ -97,7 +43,9 @@ pub(crate) fn expand_deriving_partial_eq(
         ret_ty: Path(path_local!(bool)),
         attributes: thin_vec![cx.attr_word(sym::inline, span)],
         fieldless_variants_strategy: FieldlessVariantsStrategy::Unify,
-        combine_substructure: combine_substructure(Box::new(|a, b, c| cs_eq(a, b, c))),
+        combine_substructure: combine_substructure(Box::new(|a, b, c| {
+            BlockOrExpr::new_expr(get_substructure_equality_expr(a, b, c))
+        })),
     }];
 
     let trait_def = TraitDef {
@@ -113,3 +61,156 @@ pub(crate) fn expand_deriving_partial_eq(
     };
     trait_def.expand(cx, mitem, item, push)
 }
+
+/// Generates the equality expression for a struct or enum variant when deriving
+/// `PartialEq`.
+///
+/// This function generates an expression that checks if all fields of a struct
+/// or enum variant are equal.
+/// - Scalar fields are compared first for efficiency, followed by compound
+///   fields.
+/// - If there are no fields, returns `true` (fieldless types are always equal).
+///
+/// Whether a field is considered "scalar" is determined by comparing the symbol
+/// of its type to a set of known scalar type symbols (e.g., `i32`, `u8`, etc).
+/// This check is based on the type's symbol.
+///
+/// ### Example 1
+/// ```
+/// #[derive(PartialEq)]
+/// struct i32;
+///
+/// // Here, `field_2` is of type `i32`, but since it's a user-defined type (not
+/// // the primitive), it will not be treated as scalar. The function will still
+/// // check equality of `field_2` first because the symbol matches `i32`.
+/// #[derive(PartialEq)]
+/// struct Struct {
+///     field_1: &'static str,
+///     field_2: i32,
+/// }
+/// ```
+///
+/// ### Example 2
+/// ```
+/// mod ty {
+///     pub type i32 = i32;
+/// }
+///
+/// // Here, `field_2` is of type `ty::i32`, which is a type alias for `i32`.
+/// // However, the function will not reorder the fields because the symbol for
+/// // `ty::i32` does not match the symbol for the primitive `i32`
+/// // ("ty::i32" != "i32").
+/// #[derive(PartialEq)]
+/// struct Struct {
+///     field_1: &'static str,
+///     field_2: ty::i32,
+/// }
+/// ```
+///
+/// For enums, the discriminant is compared first, then the rest of the fields.
+///
+/// # Panics
+///
+/// If called on static or all-fieldless enums/structs, which should not occur
+/// during derive expansion.
+fn get_substructure_equality_expr(
+    cx: &ExtCtxt<'_>,
+    span: Span,
+    substructure: &Substructure<'_>,
+) -> P<Expr> {
+    use SubstructureFields::*;
+
+    match substructure.fields {
+        EnumMatching(.., fields) | Struct(.., fields) => {
+            let combine = move |acc, field| {
+                let rhs = get_field_equality_expr(cx, field);
+                if let Some(lhs) = acc {
+                    // Combine the previous comparison with the current field
+                    // using logical AND.
+                    return Some(cx.expr_binary(field.span, BinOpKind::And, lhs, rhs));
+                }
+                // Start the chain with the first field's comparison.
+                Some(rhs)
+            };
+
+            // First compare scalar fields, then compound fields, combining all
+            // with logical AND.
+            return fields
+                .iter()
+                .filter(|field| !field.maybe_scalar)
+                .fold(fields.iter().filter(|field| field.maybe_scalar).fold(None, combine), combine)
+                // If there are no fields, treat as always equal.
+                .unwrap_or_else(|| cx.expr_bool(span, true));
+        }
+        EnumDiscr(disc, match_expr) => {
+            let lhs = get_field_equality_expr(cx, disc);
+            let Some(match_expr) = match_expr else {
+                return lhs;
+            };
+            // Compare the discriminant first (cheaper), then the rest of the
+            // fields.
+            return cx.expr_binary(disc.span, BinOpKind::And, lhs, match_expr.clone());
+        }
+        StaticEnum(..) => cx.dcx().span_bug(
+            span,
+            "unexpected static enum encountered during `derive(PartialEq)` expansion",
+        ),
+        StaticStruct(..) => cx.dcx().span_bug(
+            span,
+            "unexpected static struct encountered during `derive(PartialEq)` expansion",
+        ),
+        AllFieldlessEnum(..) => cx.dcx().span_bug(
+            span,
+            "unexpected all-fieldless enum encountered during `derive(PartialEq)` expansion",
+        ),
+    }
+}
+
+/// Generates an equality comparison expression for a single struct or enum
+/// field.
+///
+/// This function produces an AST expression that compares the `self` and
+/// `other` values for a field using `==`. It removes any leading references
+/// from both sides for readability. If the field is a block expression, it is
+/// wrapped in parentheses to ensure valid syntax.
+///
+/// # Panics
+///
+/// Panics if there are not exactly two arguments to compare (should be `self`
+/// and `other`).
+fn get_field_equality_expr(cx: &ExtCtxt<'_>, field: &FieldInfo) -> P<Expr> {
+    let [rhs] = &field.other_selflike_exprs[..] else {
+        cx.dcx().span_bug(field.span, "not exactly 2 arguments in `derive(PartialEq)`");
+    };
+
+    cx.expr_binary(
+        field.span,
+        BinOpKind::Eq,
+        wrap_block_expr(cx, peel_refs(&field.self_expr)),
+        wrap_block_expr(cx, peel_refs(rhs)),
+    )
+}
+
+/// Removes all leading immutable references from an expression.
+///
+/// This is used to strip away any number of leading `&` from an expression
+/// (e.g., `&&&T` becomes `T`). Only removes immutable references; mutable
+/// references are preserved.
+fn peel_refs(mut expr: &P<Expr>) -> P<Expr> {
+    while let ExprKind::AddrOf(BorrowKind::Ref, Mutability::Not, inner) = &expr.kind {
+        expr = &inner;
+    }
+    expr.clone()
+}
+
+/// Wraps a block expression in parentheses to ensure valid AST in macro
+/// expansion output.
+///
+/// If the given expression is a block, it is wrapped in parentheses; otherwise,
+/// it is returned unchanged.
+fn wrap_block_expr(cx: &ExtCtxt<'_>, expr: P<Expr>) -> P<Expr> {
+    if matches!(&expr.kind, ExprKind::Block(..)) {
+        return cx.expr_paren(expr.span, expr);
+    }
+    expr
+}
diff --git a/compiler/rustc_builtin_macros/src/deriving/cmp/partial_ord.rs b/compiler/rustc_builtin_macros/src/deriving/cmp/partial_ord.rs
index aa01da3151e..0a076dd670b 100644
--- a/compiler/rustc_builtin_macros/src/deriving/cmp/partial_ord.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/cmp/partial_ord.rs
@@ -21,7 +21,7 @@ pub(crate) fn expand_deriving_partial_ord(
 
     // Order in which to perform matching
     let discr_then_data = if let Annotatable::Item(item) = item
-        && let ItemKind::Enum(_, def, _) = &item.kind
+        && let ItemKind::Enum(_, _, def) = &item.kind
     {
         let dataful: Vec<bool> = def.variants.iter().map(|v| !v.data.fields().is_empty()).collect();
         match dataful.iter().filter(|&&b| b).count() {
diff --git a/compiler/rustc_builtin_macros/src/deriving/coerce_pointee.rs b/compiler/rustc_builtin_macros/src/deriving/coerce_pointee.rs
index 446d8afeedd..0794192621a 100644
--- a/compiler/rustc_builtin_macros/src/deriving/coerce_pointee.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/coerce_pointee.rs
@@ -30,7 +30,7 @@ pub(crate) fn expand_deriving_coerce_pointee(
     item.visit_with(&mut DetectNonGenericPointeeAttr { cx });
 
     let (name_ident, generics) = if let Annotatable::Item(aitem) = item
-        && let ItemKind::Struct(ident, struct_data, g) = &aitem.kind
+        && let ItemKind::Struct(ident, g, struct_data) = &aitem.kind
     {
         if !matches!(
             struct_data,
diff --git a/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs b/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs
index 9aa53f9e4f7..6dd3adf750d 100644
--- a/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs
@@ -284,6 +284,7 @@ pub(crate) struct FieldInfo {
     /// The expressions corresponding to references to this field in
     /// the other selflike arguments.
     pub other_selflike_exprs: Vec<P<Expr>>,
+    pub maybe_scalar: bool,
 }
 
 #[derive(Copy, Clone)]
@@ -488,7 +489,7 @@ impl<'a> TraitDef<'a> {
                 );
 
                 let newitem = match &item.kind {
-                    ast::ItemKind::Struct(ident, struct_def, generics) => self.expand_struct_def(
+                    ast::ItemKind::Struct(ident, generics, struct_def) => self.expand_struct_def(
                         cx,
                         struct_def,
                         *ident,
@@ -496,7 +497,7 @@ impl<'a> TraitDef<'a> {
                         from_scratch,
                         is_packed,
                     ),
-                    ast::ItemKind::Enum(ident, enum_def, generics) => {
+                    ast::ItemKind::Enum(ident, generics, enum_def) => {
                         // We ignore `is_packed` here, because `repr(packed)`
                         // enums cause an error later on.
                         //
@@ -504,7 +505,7 @@ impl<'a> TraitDef<'a> {
                         // downstream in blatantly illegal code, so it is fine.
                         self.expand_enum_def(cx, enum_def, *ident, generics, from_scratch)
                     }
-                    ast::ItemKind::Union(ident, struct_def, generics) => {
+                    ast::ItemKind::Union(ident, generics, struct_def) => {
                         if self.supports_unions {
                             self.expand_struct_def(
                                 cx,
@@ -1220,7 +1221,8 @@ impl<'a> MethodDef<'a> {
 
             let self_expr = discr_exprs.remove(0);
             let other_selflike_exprs = discr_exprs;
-            let discr_field = FieldInfo { span, name: None, self_expr, other_selflike_exprs };
+            let discr_field =
+                FieldInfo { span, name: None, self_expr, other_selflike_exprs, maybe_scalar: true };
 
             let discr_let_stmts: ThinVec<_> = iter::zip(&discr_idents, &selflike_args)
                 .map(|(&ident, selflike_arg)| {
@@ -1533,6 +1535,7 @@ impl<'a> TraitDef<'a> {
                     name: struct_field.ident,
                     self_expr,
                     other_selflike_exprs,
+                    maybe_scalar: struct_field.ty.peel_refs().kind.maybe_scalar(),
                 }
             })
             .collect()
diff --git a/compiler/rustc_builtin_macros/src/deriving/mod.rs b/compiler/rustc_builtin_macros/src/deriving/mod.rs
index 50e7b989ed8..e45d09b5796 100644
--- a/compiler/rustc_builtin_macros/src/deriving/mod.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/mod.rs
@@ -57,7 +57,7 @@ impl MultiItemModifier for BuiltinDerive {
         let mut items = Vec::new();
         match item {
             Annotatable::Stmt(stmt) => {
-                if let ast::StmtKind::Item(item) = stmt.into_inner().kind {
+                if let ast::StmtKind::Item(item) = stmt.kind {
                     (self.0)(
                         ecx,
                         span,
diff --git a/compiler/rustc_builtin_macros/src/errors.rs b/compiler/rustc_builtin_macros/src/errors.rs
index b28f7d312d9..3a2e96a5e5a 100644
--- a/compiler/rustc_builtin_macros/src/errors.rs
+++ b/compiler/rustc_builtin_macros/src/errors.rs
@@ -110,13 +110,6 @@ pub(crate) struct ProcMacro {
 }
 
 #[derive(Diagnostic)]
-#[diag(builtin_macros_non_abi)]
-pub(crate) struct NonABI {
-    #[primary_span]
-    pub(crate) span: Span,
-}
-
-#[derive(Diagnostic)]
 #[diag(builtin_macros_trace_macros)]
 pub(crate) struct TraceMacros {
     #[primary_span]
@@ -188,14 +181,6 @@ mod autodiff {
     }
 
     #[derive(Diagnostic)]
-    #[diag(builtin_macros_autodiff_mode)]
-    pub(crate) struct AutoDiffInvalidMode {
-        #[primary_span]
-        pub(crate) span: Span,
-        pub(crate) mode: String,
-    }
-
-    #[derive(Diagnostic)]
     #[diag(builtin_macros_autodiff_width)]
     pub(crate) struct AutoDiffInvalidWidth {
         #[primary_span]
@@ -789,52 +774,13 @@ pub(crate) struct AsmModifierInvalid {
 }
 
 #[derive(Diagnostic)]
-#[diag(builtin_macros_asm_requires_template)]
-pub(crate) struct AsmRequiresTemplate {
-    #[primary_span]
-    pub(crate) span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(builtin_macros_asm_expected_comma)]
-pub(crate) struct AsmExpectedComma {
-    #[primary_span]
-    #[label]
-    pub(crate) span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(builtin_macros_asm_expected_string_literal)]
-pub(crate) struct AsmExpectedStringLiteral {
+#[diag(builtin_macros_asm_attribute_not_supported)]
+pub(crate) struct AsmAttributeNotSupported {
     #[primary_span]
-    #[label]
     pub(crate) span: Span,
 }
 
 #[derive(Diagnostic)]
-#[diag(builtin_macros_asm_underscore_input)]
-pub(crate) struct AsmUnderscoreInput {
-    #[primary_span]
-    pub(crate) span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(builtin_macros_asm_sym_no_path)]
-pub(crate) struct AsmSymNoPath {
-    #[primary_span]
-    pub(crate) span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(builtin_macros_asm_expected_other)]
-pub(crate) struct AsmExpectedOther {
-    #[primary_span]
-    #[label(builtin_macros_asm_expected_other)]
-    pub(crate) span: Span,
-    pub(crate) is_inline_asm: bool,
-}
-
-#[derive(Diagnostic)]
 #[diag(builtin_macros_asm_duplicate_arg)]
 pub(crate) struct AsmDuplicateArg {
     #[primary_span]
@@ -926,16 +872,6 @@ pub(crate) struct AsmUnsupportedOption {
 }
 
 #[derive(Diagnostic)]
-#[diag(builtin_macros_asm_unsupported_operand)]
-pub(crate) struct AsmUnsupportedOperand<'a> {
-    #[primary_span]
-    #[label]
-    pub(crate) span: Span,
-    pub(crate) symbol: &'a str,
-    pub(crate) macro_name: &'static str,
-}
-
-#[derive(Diagnostic)]
 #[diag(builtin_macros_asm_unsupported_clobber_abi)]
 pub(crate) struct AsmUnsupportedClobberAbi {
     #[primary_span]
@@ -958,13 +894,6 @@ pub(crate) struct TestRunnerNargs {
 }
 
 #[derive(Diagnostic)]
-#[diag(builtin_macros_expected_register_class_or_explicit_register)]
-pub(crate) struct ExpectedRegisterClassOrExplicitRegister {
-    #[primary_span]
-    pub(crate) span: Span,
-}
-
-#[derive(Diagnostic)]
 #[diag(builtin_macros_expected_comma_in_list)]
 pub(crate) struct ExpectedCommaInList {
     #[primary_span]
@@ -1027,3 +956,12 @@ pub(crate) struct NonGenericPointee {
     #[primary_span]
     pub span: Span,
 }
+
+#[derive(Diagnostic)]
+#[diag(builtin_macros_expected_other)]
+pub(crate) struct AsmExpectedOther {
+    #[primary_span]
+    #[label(builtin_macros_expected_other)]
+    pub(crate) span: Span,
+    pub(crate) is_inline_asm: bool,
+}
diff --git a/compiler/rustc_builtin_macros/src/iter.rs b/compiler/rustc_builtin_macros/src/iter.rs
new file mode 100644
index 00000000000..7ad83903a1b
--- /dev/null
+++ b/compiler/rustc_builtin_macros/src/iter.rs
@@ -0,0 +1,53 @@
+use rustc_ast::ptr::P;
+use rustc_ast::tokenstream::TokenStream;
+use rustc_ast::{CoroutineKind, DUMMY_NODE_ID, Expr, ast, token};
+use rustc_errors::PResult;
+use rustc_expand::base::{self, DummyResult, ExpandResult, ExtCtxt, MacroExpanderResult};
+use rustc_span::Span;
+
+pub(crate) fn expand<'cx>(
+    cx: &'cx mut ExtCtxt<'_>,
+    sp: Span,
+    tts: TokenStream,
+) -> MacroExpanderResult<'cx> {
+    let closure = match parse_closure(cx, sp, tts) {
+        Ok(parsed) => parsed,
+        Err(err) => {
+            return ExpandResult::Ready(DummyResult::any(sp, err.emit()));
+        }
+    };
+
+    ExpandResult::Ready(base::MacEager::expr(closure))
+}
+
+fn parse_closure<'a>(
+    cx: &mut ExtCtxt<'a>,
+    span: Span,
+    stream: TokenStream,
+) -> PResult<'a, P<Expr>> {
+    let mut closure_parser = cx.new_parser_from_tts(stream);
+
+    let coroutine_kind = Some(CoroutineKind::Gen {
+        span,
+        closure_id: DUMMY_NODE_ID,
+        return_impl_trait_id: DUMMY_NODE_ID,
+    });
+
+    let mut closure = closure_parser.parse_expr()?;
+    match &mut closure.kind {
+        ast::ExprKind::Closure(c) => {
+            if let Some(kind) = c.coroutine_kind {
+                cx.dcx().span_err(kind.span(), "only plain closures allowed in `iter!`");
+            }
+            c.coroutine_kind = coroutine_kind;
+            if closure_parser.token != token::Eof {
+                closure_parser.unexpected()?;
+            }
+            Ok(closure)
+        }
+        _ => {
+            cx.dcx().span_err(closure.span, "`iter!` body must be a closure");
+            Err(closure_parser.unexpected().unwrap_err())
+        }
+    }
+}
diff --git a/compiler/rustc_builtin_macros/src/lib.rs b/compiler/rustc_builtin_macros/src/lib.rs
index 9cd4d17059a..9e7d0ec9e81 100644
--- a/compiler/rustc_builtin_macros/src/lib.rs
+++ b/compiler/rustc_builtin_macros/src/lib.rs
@@ -5,23 +5,20 @@
 #![allow(internal_features)]
 #![allow(rustc::diagnostic_outside_of_impl)]
 #![allow(rustc::untranslatable_diagnostic)]
+#![cfg_attr(not(bootstrap), feature(autodiff))]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(assert_matches)]
-#![feature(autodiff)]
 #![feature(box_patterns)]
 #![feature(decl_macro)]
 #![feature(if_let_guard)]
 #![feature(proc_macro_internals)]
 #![feature(proc_macro_quote)]
 #![feature(rustdoc_internals)]
-#![feature(string_from_utf8_lossy_owned)]
 #![feature(try_blocks)]
 #![recursion_limit = "256"]
 // tidy-alphabetical-end
 
-extern crate proc_macro;
-
 use std::sync::Arc;
 
 use rustc_expand::base::{MacroExpanderFn, ResolverExpand, SyntaxExtensionKind};
@@ -49,6 +46,7 @@ mod errors;
 mod format;
 mod format_foreign;
 mod global_allocator;
+mod iter;
 mod log_syntax;
 mod pattern_type;
 mod source_util;
@@ -97,6 +95,7 @@ pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) {
         include: source_util::expand_include,
         include_bytes: source_util::expand_include_bytes,
         include_str: source_util::expand_include_str,
+        iter: iter::expand,
         line: source_util::expand_line,
         log_syntax: log_syntax::expand_log_syntax,
         module_path: source_util::expand_mod,
@@ -112,7 +111,8 @@ pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) {
 
     register_attr! {
         alloc_error_handler: alloc_error_handler::expand,
-        autodiff: autodiff::expand,
+        autodiff_forward: autodiff::expand_forward,
+        autodiff_reverse: autodiff::expand_reverse,
         bench: test::expand_bench,
         cfg_accessible: cfg_accessible::Expander,
         cfg_eval: cfg_eval::expand,
@@ -139,7 +139,7 @@ pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) {
         CoercePointee: coerce_pointee::expand_deriving_coerce_pointee,
     }
 
-    let client = proc_macro::bridge::client::Client::expand1(proc_macro::quote);
+    let client = rustc_proc_macro::bridge::client::Client::expand1(rustc_proc_macro::quote);
     register(sym::quote, SyntaxExtensionKind::Bang(Arc::new(BangProcMacro { client })));
     let requires = SyntaxExtensionKind::Attr(Arc::new(contracts::ExpandRequires));
     register(sym::contracts_requires, requires);
diff --git a/compiler/rustc_builtin_macros/src/pattern_type.rs b/compiler/rustc_builtin_macros/src/pattern_type.rs
index 3529e5525fc..b61af0b0aaa 100644
--- a/compiler/rustc_builtin_macros/src/pattern_type.rs
+++ b/compiler/rustc_builtin_macros/src/pattern_type.rs
@@ -30,14 +30,12 @@ fn parse_pat_ty<'a>(cx: &mut ExtCtxt<'a>, stream: TokenStream) -> PResult<'a, (P
 
     let pat = pat_to_ty_pat(
         cx,
-        parser
-            .parse_pat_no_top_guard(
-                None,
-                RecoverComma::No,
-                RecoverColon::No,
-                CommaRecoveryMode::EitherTupleOrPipe,
-            )?
-            .into_inner(),
+        *parser.parse_pat_no_top_guard(
+            None,
+            RecoverComma::No,
+            RecoverColon::No,
+            CommaRecoveryMode::EitherTupleOrPipe,
+        )?,
     );
 
     if parser.token != token::Eof {
@@ -58,9 +56,9 @@ fn pat_to_ty_pat(cx: &mut ExtCtxt<'_>, pat: ast::Pat) -> P<TyPat> {
             end.map(|value| P(AnonConst { id: DUMMY_NODE_ID, value })),
             include_end,
         ),
-        ast::PatKind::Or(variants) => TyPatKind::Or(
-            variants.into_iter().map(|pat| pat_to_ty_pat(cx, pat.into_inner())).collect(),
-        ),
+        ast::PatKind::Or(variants) => {
+            TyPatKind::Or(variants.into_iter().map(|pat| pat_to_ty_pat(cx, *pat)).collect())
+        }
         ast::PatKind::Err(guar) => TyPatKind::Err(guar),
         _ => TyPatKind::Err(cx.dcx().span_err(pat.span, "pattern not supported in pattern types")),
     };
diff --git a/compiler/rustc_builtin_macros/src/proc_macro_harness.rs b/compiler/rustc_builtin_macros/src/proc_macro_harness.rs
index a91f2d38a93..daf480a9ce4 100644
--- a/compiler/rustc_builtin_macros/src/proc_macro_harness.rs
+++ b/compiler/rustc_builtin_macros/src/proc_macro_harness.rs
@@ -354,30 +354,28 @@ fn mk_decls(cx: &mut ExtCtxt<'_>, macros: &[ProcMacro]) -> P<ast::Item> {
         })
         .collect();
 
-    let decls_static = cx
-        .item_static(
+    let mut decls_static = cx.item_static(
+        span,
+        Ident::new(sym::_DECLS, span),
+        cx.ty_ref(
             span,
-            Ident::new(sym::_DECLS, span),
-            cx.ty_ref(
+            cx.ty(
                 span,
-                cx.ty(
-                    span,
-                    ast::TyKind::Slice(
-                        cx.ty_path(cx.path(span, vec![proc_macro, bridge, client, proc_macro_ty])),
-                    ),
+                ast::TyKind::Slice(
+                    cx.ty_path(cx.path(span, vec![proc_macro, bridge, client, proc_macro_ty])),
                 ),
-                None,
-                ast::Mutability::Not,
             ),
+            None,
             ast::Mutability::Not,
-            cx.expr_array_ref(span, decls),
-        )
-        .map(|mut i| {
-            i.attrs.push(cx.attr_word(sym::rustc_proc_macro_decls, span));
-            i.attrs.push(cx.attr_word(sym::used, span));
-            i.attrs.push(cx.attr_nested_word(sym::allow, sym::deprecated, span));
-            i
-        });
+        ),
+        ast::Mutability::Not,
+        cx.expr_array_ref(span, decls),
+    );
+    decls_static.attrs.extend([
+        cx.attr_word(sym::rustc_proc_macro_decls, span),
+        cx.attr_word(sym::used, span),
+        cx.attr_nested_word(sym::allow, sym::deprecated, span),
+    ]);
 
     let block = cx.expr_block(
         cx.block(span, thin_vec![cx.stmt_item(span, krate), cx.stmt_item(span, decls_static)]),
diff --git a/compiler/rustc_builtin_macros/src/test.rs b/compiler/rustc_builtin_macros/src/test.rs
index 1cef4f9514c..b439fa34f5b 100644
--- a/compiler/rustc_builtin_macros/src/test.rs
+++ b/compiler/rustc_builtin_macros/src/test.rs
@@ -40,7 +40,7 @@ pub(crate) fn expand_test_case(
     let (mut item, is_stmt) = match anno_item {
         Annotatable::Item(item) => (item, false),
         Annotatable::Stmt(stmt) if let ast::StmtKind::Item(_) = stmt.kind => {
-            if let ast::StmtKind::Item(i) = stmt.into_inner().kind {
+            if let ast::StmtKind::Item(i) = stmt.kind {
                 (i, true)
             } else {
                 unreachable!()
@@ -120,11 +120,7 @@ pub(crate) fn expand_test_or_bench(
         Annotatable::Item(i) => (i, false),
         Annotatable::Stmt(stmt) if matches!(stmt.kind, ast::StmtKind::Item(_)) => {
             // FIXME: Use an 'if let' guard once they are implemented
-            if let ast::StmtKind::Item(i) = stmt.into_inner().kind {
-                (i, true)
-            } else {
-                unreachable!()
-            }
+            if let ast::StmtKind::Item(i) = stmt.kind { (i, true) } else { unreachable!() }
         }
         other => {
             not_testable_error(cx, attr_sp, None);
@@ -381,10 +377,7 @@ pub(crate) fn expand_test_or_bench(
                 .into(),
             ),
         );
-    test_const = test_const.map(|mut tc| {
-        tc.vis.kind = ast::VisibilityKind::Public;
-        tc
-    });
+    test_const.vis.kind = ast::VisibilityKind::Public;
 
     // extern crate test
     let test_extern =
diff --git a/compiler/rustc_builtin_macros/src/test_harness.rs b/compiler/rustc_builtin_macros/src/test_harness.rs
index 56a67b0534d..0bc313cbdac 100644
--- a/compiler/rustc_builtin_macros/src/test_harness.rs
+++ b/compiler/rustc_builtin_macros/src/test_harness.rs
@@ -128,9 +128,7 @@ impl<'a> MutVisitor for TestHarnessGenerator<'a> {
         c.items.push(mk_main(&mut self.cx));
     }
 
-    fn visit_item(&mut self, item: &mut P<ast::Item>) {
-        let item = &mut **item;
-
+    fn visit_item(&mut self, item: &mut ast::Item) {
         if let Some(name) = get_test_name(&item) {
             debug!("this is a test item");
 
@@ -193,7 +191,7 @@ struct EntryPointCleaner<'a> {
 }
 
 impl<'a> MutVisitor for EntryPointCleaner<'a> {
-    fn visit_item(&mut self, item: &mut P<ast::Item>) {
+    fn visit_item(&mut self, item: &mut ast::Item) {
         self.depth += 1;
         ast::mut_visit::walk_item(self, item);
         self.depth -= 1;
diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/main.yml b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml
index 6fd288d195c..d92e0fdce99 100644
--- a/compiler/rustc_codegen_cranelift/.github/workflows/main.yml
+++ b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml
@@ -34,9 +34,9 @@ jobs:
     - name: Rustfmt
       run: |
         cargo fmt --check
-        rustfmt --check build_system/main.rs
-        rustfmt --check example/*
-        rustfmt --check scripts/*.rs
+        rustfmt --check --edition 2024 build_system/main.rs
+        rustfmt --check --edition 2024 example/*
+        rustfmt --check --edition 2024 scripts/*.rs
 
 
   test:
diff --git a/compiler/rustc_codegen_cranelift/Cargo.lock b/compiler/rustc_codegen_cranelift/Cargo.lock
index e5f1896b923..a906bec8b7e 100644
--- a/compiler/rustc_codegen_cranelift/Cargo.lock
+++ b/compiler/rustc_codegen_cranelift/Cargo.lock
@@ -43,39 +43,42 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-assembler-x64"
-version = "0.118.0"
+version = "0.120.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3e4b56ebe316895d3fa37775d0a87b0c889cc933f5c8b253dbcc7c7bcb7fe7e4"
+checksum = "9ff8e35182c7372df00447cb90a04e584e032c42b9b9b6e8c50ddaaf0d7900d5"
 dependencies = [
  "cranelift-assembler-x64-meta",
 ]
 
 [[package]]
 name = "cranelift-assembler-x64-meta"
-version = "0.118.0"
+version = "0.120.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95cabbc01dfbd7dcd6c329ca44f0212910309c221797ac736a67a5bc8857fe1b"
+checksum = "14220f9c2698015c3b94dc6b84ae045c1c45509ddc406e43c6139252757fdb7a"
+dependencies = [
+ "cranelift-srcgen",
+]
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.118.0"
+version = "0.120.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76ffe46df300a45f1dc6f609dc808ce963f0e3a2e971682c479a2d13e3b9b8ef"
+checksum = "d372ef2777ceefd75829e1390211ac240e9196bc60699218f7ea2419038288ee"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-bitset"
-version = "0.118.0"
+version = "0.120.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b265bed7c51e1921fdae6419791d31af77d33662ee56d7b0fa0704dc8d231cab"
+checksum = "56323783e423818fa89ce8078e90a3913d2a6e0810399bfce8ebd7ee87baa81f"
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.118.0"
+version = "0.120.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e606230a7e3a6897d603761baee0d19f88d077f17b996bb5089488a29ae96e41"
+checksum = "74ffb780aab6186c6e9ba26519654b1ac55a09c0a866f6088a4efbbd84da68ed"
 dependencies = [
  "bumpalo",
  "cranelift-assembler-x64",
@@ -98,43 +101,44 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.118.0"
+version = "0.120.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a63bffafc23bc60969ad528e138788495999d935f0adcfd6543cb151ca8637d"
+checksum = "c23ef13814d3b39c869650d5961128cbbecad83fbdff4e6836a03ecf6862d7ed"
 dependencies = [
- "cranelift-assembler-x64",
+ "cranelift-assembler-x64-meta",
  "cranelift-codegen-shared",
+ "cranelift-srcgen",
 ]
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.118.0"
+version = "0.120.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af50281b67324b58e843170a6a5943cf6d387c06f7eeacc9f5696e4ab7ae7d7e"
+checksum = "b9f623300657679f847803ce80811454bfff89cea4f6bf684be5c468d4a73631"
 
 [[package]]
 name = "cranelift-control"
-version = "0.118.0"
+version = "0.120.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c20c1b38d1abfbcebb0032e497e71156c0e3b8dcb3f0a92b9863b7bcaec290c"
+checksum = "31f4168af69989aa6b91fab46799ed4df6096f3209f4a6c8fb4358f49c60188f"
 dependencies = [
  "arbitrary",
 ]
 
 [[package]]
 name = "cranelift-entity"
-version = "0.118.0"
+version = "0.120.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c2c67d95507c51b4a1ff3f3555fe4bfec36b9e13c1b684ccc602736f5d5f4a2"
+checksum = "ca6fa9bae1c8de26d71ac2162f069447610fd91e7780cb480ee0d76ac81eabb8"
 dependencies = [
  "cranelift-bitset",
 ]
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.118.0"
+version = "0.120.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e002691cc69c38b54fc7ec93e5be5b744f627d027031d991cc845d1d512d0ce"
+checksum = "b8219205608aa0b0e6769b580284a7e055c7e0c323c1041cde7ca078add3e412"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -144,15 +148,15 @@ dependencies = [
 
 [[package]]
 name = "cranelift-isle"
-version = "0.118.0"
+version = "0.120.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e93588ed1796cbcb0e2ad160403509e2c5d330d80dd6e0014ac6774c7ebac496"
+checksum = "588d0c5964f10860b04043e55aab26d7f7a206b0fd4f10c5260e8aa5773832bd"
 
 [[package]]
 name = "cranelift-jit"
-version = "0.118.0"
+version = "0.120.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17f6682f0b193d6b7873cc8e7ed67e8776a8a26f50eeabf88534e9be618b9a03"
+checksum = "56bd917ddc524f84f4066f954062875bdfc0dffea068ee94e906d98de5ac7c33"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -170,9 +174,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.118.0"
+version = "0.120.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff19784c6de05116e63e6a34791012bd927b2a4eac56233039c46f1b6a4edac8"
+checksum = "68a03c057d8a992e06596c871341e446af43ff9224f941e5b8adea39137a5391"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -181,9 +185,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-native"
-version = "0.118.0"
+version = "0.120.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5b09bdd6407bf5d89661b80cf926ce731c9e8cc184bf49102267a2369a8358e"
+checksum = "19ed3c94cb97b14f92b6a94a1d45ef8c851f6a2ad9114e5d91d233f7da638fed"
 dependencies = [
  "cranelift-codegen",
  "libc",
@@ -192,9 +196,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.118.0"
+version = "0.120.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "685e8661a30d1cb69509f589ac643adeee79c5f63c0da316431b9fad29e6d3b4"
+checksum = "a64dacef362a69375a604f6636e5e9a174fb96dba3b273646fcd9fa85c1d0997"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -206,6 +210,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "cranelift-srcgen"
+version = "0.120.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85256fac1519a7d25a040c1d850fba67478f3f021ad5fdf738ba4425ee862dbf"
+
+[[package]]
 name = "crc32fast"
 version = "1.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -331,9 +341,9 @@ dependencies = [
 
 [[package]]
 name = "regalloc2"
-version = "0.11.1"
+version = "0.12.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "145c1c267e14f20fb0f88aa76a1c5ffec42d592c1d28b3cd9148ae35916158d3"
+checksum = "5216b1837de2149f8bc8e6d5f88a9326b63b8c836ed58ce4a0a29ec736a59734"
 dependencies = [
  "allocator-api2",
  "bumpalo",
@@ -436,9 +446,9 @@ checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
 
 [[package]]
 name = "wasmtime-jit-icache-coherence"
-version = "31.0.0"
+version = "33.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a54f6c6c7e9d7eeee32dfcc10db7f29d505ee7dd28d00593ea241d5f70698e64"
+checksum = "175e924dbc944c185808466d1e90b5a7feb610f3b9abdfe26f8ee25fd1086d1c"
 dependencies = [
  "anyhow",
  "cfg-if",
diff --git a/compiler/rustc_codegen_cranelift/Cargo.toml b/compiler/rustc_codegen_cranelift/Cargo.toml
index 08b60de14c1..94fcbd0a502 100644
--- a/compiler/rustc_codegen_cranelift/Cargo.toml
+++ b/compiler/rustc_codegen_cranelift/Cargo.toml
@@ -8,12 +8,12 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { version = "0.118.0", default-features = false, features = ["std", "timing", "unwind", "all-native-arch"] }
-cranelift-frontend = { version = "0.118.0" }
-cranelift-module = { version = "0.118.0" }
-cranelift-native = { version = "0.118.0" }
-cranelift-jit = { version = "0.118.0", optional = true }
-cranelift-object = { version = "0.118.0" }
+cranelift-codegen = { version = "0.120.0", default-features = false, features = ["std", "timing", "unwind", "all-native-arch"] }
+cranelift-frontend = { version = "0.120.0" }
+cranelift-module = { version = "0.120.0" }
+cranelift-native = { version = "0.120.0" }
+cranelift-jit = { version = "0.120.0", optional = true }
+cranelift-object = { version = "0.120.0" }
 target-lexicon = "0.13"
 gimli = { version = "0.31", default-features = false, features = ["write"] }
 object = { version = "0.36", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
@@ -24,12 +24,12 @@ smallvec = "1.8.1"
 
 [patch.crates-io]
 # Uncomment to use an unreleased version of cranelift
-#cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-31.0.0", version = "0.118.0" }
-#cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-31.0.0", version = "0.118.0" }
-#cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-31.0.0", version = "0.118.0" }
-#cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-31.0.0", version = "0.118.0" }
-#cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-31.0.0", version = "0.118.0" }
-#cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-31.0.0", version = "0.118.0" }
+#cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" }
+#cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" }
+#cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" }
+#cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" }
+#cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" }
+#cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" }
 
 # Uncomment to use local checkout of cranelift
 #cranelift-codegen = { path = "../wasmtime/cranelift/codegen" }
diff --git a/compiler/rustc_codegen_cranelift/Readme.md b/compiler/rustc_codegen_cranelift/Readme.md
index 28edb5795ce..4d1e4d843ff 100644
--- a/compiler/rustc_codegen_cranelift/Readme.md
+++ b/compiler/rustc_codegen_cranelift/Readme.md
@@ -7,7 +7,7 @@ If not please open an issue.
 
 ## Download using Rustup
 
-The Cranelift codegen backend is distributed in nightly builds on Linux and x86_64 macOS. If you want to
+The Cranelift codegen backend is distributed in nightly builds on Linux, macOS and x86_64 Windows. If you want to
 install it using Rustup, you can do that by running:
 
 ```bash
@@ -79,7 +79,7 @@ For more docs on how to build and test see [build_system/usage.txt](build_system
 Not all targets are available as rustup component for nightly. See notes in the platform support matrix.
 
 [^xcoff]: XCOFF object file format is not supported.
-[^no-rustup]: Not available as rustup component for nightly. You can build it yourself.
+[^no-rustup]: Not available as [rustup component for nightly](https://rust-lang.github.io/rustup-components-history/). You can build it yourself.
 
 ## Usage
 
diff --git a/compiler/rustc_codegen_cranelift/build_system/build_backend.rs b/compiler/rustc_codegen_cranelift/build_system/build_backend.rs
index 72bc422523d..bf7cf1c0a34 100644
--- a/compiler/rustc_codegen_cranelift/build_system/build_backend.rs
+++ b/compiler/rustc_codegen_cranelift/build_system/build_backend.rs
@@ -18,7 +18,7 @@ pub(crate) fn build_backend(
     let mut cmd = CG_CLIF.build(&bootstrap_host_compiler, dirs);
 
     let mut rustflags = rustflags_from_env("RUSTFLAGS");
-    rustflags.push("-Zallow-features=rustc_private".to_owned());
+    rustflags.push("-Zallow-features=rustc_private,f16,f128".to_owned());
     rustflags_to_cmd_env(&mut cmd, "RUSTFLAGS", &rustflags);
 
     if env::var("CG_CLIF_EXPENSIVE_CHECKS").is_ok() {
diff --git a/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs b/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs
index a73e3c87d43..00955998e70 100644
--- a/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs
+++ b/compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs
@@ -168,7 +168,8 @@ fn build_llvm_sysroot_for_triple(compiler: Compiler) -> SysrootTarget {
         let file_name_str = file.file_name().unwrap().to_str().unwrap();
         if (file_name_str.contains("rustc_")
             && !file_name_str.contains("rustc_std_workspace_")
-            && !file_name_str.contains("rustc_demangle"))
+            && !file_name_str.contains("rustc_demangle")
+            && !file_name_str.contains("rustc_literal_escaper"))
             || file_name_str.contains("chalk")
             || file_name_str.contains("tracing")
             || file_name_str.contains("regex")
@@ -234,7 +235,7 @@ fn build_clif_sysroot_for_triple(
     compiler.rustflags.extend(rustflags);
     let mut build_cmd = STANDARD_LIBRARY.build(&compiler, dirs);
     build_cmd.arg("--release");
-    build_cmd.arg("--features").arg("backtrace panic-unwind compiler-builtins-no-f16-f128");
+    build_cmd.arg("--features").arg("backtrace panic-unwind");
     build_cmd.arg(format!("-Zroot-dir={}", STDLIB_SRC.to_path(dirs).display()));
     build_cmd.env("CARGO_PROFILE_RELEASE_DEBUG", "true");
     build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif");
diff --git a/compiler/rustc_codegen_cranelift/build_system/utils.rs b/compiler/rustc_codegen_cranelift/build_system/utils.rs
index f2399768459..d9807155a3d 100644
--- a/compiler/rustc_codegen_cranelift/build_system/utils.rs
+++ b/compiler/rustc_codegen_cranelift/build_system/utils.rs
@@ -213,11 +213,13 @@ pub(crate) fn copy_dir_recursively(from: &Path, to: &Path) {
         if filename == "." || filename == ".." {
             continue;
         }
+        let src = from.join(&filename);
+        let dst = to.join(&filename);
         if entry.metadata().unwrap().is_dir() {
-            fs::create_dir(to.join(&filename)).unwrap();
-            copy_dir_recursively(&from.join(&filename), &to.join(&filename));
+            fs::create_dir(&dst).unwrap_or_else(|e| panic!("failed to create {dst:?}: {e}"));
+            copy_dir_recursively(&src, &dst);
         } else {
-            fs::copy(from.join(&filename), to.join(&filename)).unwrap();
+            fs::copy(&src, &dst).unwrap_or_else(|e| panic!("failed to copy {src:?}->{dst:?}: {e}"));
         }
     }
 }
diff --git a/compiler/rustc_codegen_cranelift/docs/usage.md b/compiler/rustc_codegen_cranelift/docs/usage.md
index dbe36109f83..9dcfee4f535 100644
--- a/compiler/rustc_codegen_cranelift/docs/usage.md
+++ b/compiler/rustc_codegen_cranelift/docs/usage.md
@@ -47,7 +47,7 @@ These are a few functions that allow you to easily run rust code from the shell
 
 ```bash
 function jit_naked() {
-    echo "$@" | $cg_clif_dir/dist/rustc-clif - -Zunstable-options -Cllvm-args=jit-mode-Cprefer-dynamic
+    echo "$@" | $cg_clif_dir/dist/rustc-clif - -Zunstable-options -Cllvm-args=jit-mode -Cprefer-dynamic
 }
 
 function jit() {
diff --git a/compiler/rustc_codegen_cranelift/example/mini_core.rs b/compiler/rustc_codegen_cranelift/example/mini_core.rs
index 6e345b2a6fd..1dc799c0aee 100644
--- a/compiler/rustc_codegen_cranelift/example/mini_core.rs
+++ b/compiler/rustc_codegen_cranelift/example/mini_core.rs
@@ -521,10 +521,28 @@ fn panic_cannot_unwind() -> ! {
 }
 
 #[lang = "eh_personality"]
-fn eh_personality() -> ! {
+// FIXME personality signature depends on target
+fn eh_personality(
+    _version: i32,
+    _actions: i32,
+    _exception_class: u64,
+    _exception_object: *mut (),
+    _context: *mut (),
+) -> i32 {
     loop {}
 }
 
+#[lang = "panic_in_cleanup"]
+fn panic_in_cleanup() -> ! {
+    loop {}
+}
+
+#[cfg(all(unix, not(target_vendor = "apple")))]
+#[link(name = "gcc_s")]
+extern "C" {
+    fn _Unwind_Resume(exc: *mut ()) -> !;
+}
+
 #[lang = "drop_in_place"]
 #[allow(unconditional_recursion)]
 pub unsafe fn drop_in_place<T: ?Sized>(to_drop: *mut T) {
diff --git a/compiler/rustc_codegen_cranelift/example/neon.rs b/compiler/rustc_codegen_cranelift/example/neon.rs
index 69ce17d3d75..704f866e2c4 100644
--- a/compiler/rustc_codegen_cranelift/example/neon.rs
+++ b/compiler/rustc_codegen_cranelift/example/neon.rs
@@ -233,7 +233,7 @@ unsafe fn test_vaddvq_f32() {
 
 #[cfg(target_arch = "aarch64")]
 unsafe fn test_vrndnq_f32() {
-    // AArch64 llvm intrinsic: llvm.aarch64.neon.frintn.v4f32
+    // llvm intrinsic: llvm.roundeven.v4f32
     let a = f32x4::from([0.1, -1.9, 4.5, 5.5]);
     let e = f32x4::from([0., -2., 4., 6.]);
     let r: f32x4 = transmute(vrndnq_f32(transmute(a)));
diff --git a/compiler/rustc_codegen_cranelift/example/std_example.rs b/compiler/rustc_codegen_cranelift/example/std_example.rs
index 2d9de2a5b8d..5d83066cffb 100644
--- a/compiler/rustc_codegen_cranelift/example/std_example.rs
+++ b/compiler/rustc_codegen_cranelift/example/std_example.rs
@@ -166,6 +166,7 @@ fn main() {
         enum Never {}
     }
 
+    #[cfg(not(target_arch = "s390x"))] // s390x doesn't have vector instructions enabled by default
     foo(I64X2([0, 0]));
 
     transmute_wide_pointer();
@@ -203,9 +204,11 @@ fn rust_call_abi() {
     rust_call_abi_callee((1, 2));
 }
 
+#[cfg_attr(target_arch = "s390x", allow(dead_code))]
 #[repr(simd)]
 struct I64X2([i64; 2]);
 
+#[cfg_attr(target_arch = "s390x", allow(dead_code))]
 #[allow(improper_ctypes_definitions)]
 extern "C" fn foo(_a: I64X2) {}
 
diff --git a/compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch b/compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch
index d7e3b11127c..38bb43f8204 100644
--- a/compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch
+++ b/compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch
@@ -37,6 +37,15 @@ diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs
 index bf2b6d59f88..d5ccce03bbf 100644
 --- a/library/core/src/sync/atomic.rs
 +++ b/library/core/src/sync/atomic.rs
+@@ -300,8 +300,6 @@ impl_atomic_primitive!(AtomicI32(i32), size("32"), align(4));
+ impl_atomic_primitive!(AtomicU32(u32), size("32"), align(4));
+ impl_atomic_primitive!(AtomicI64(i64), size("64"), align(8));
+ impl_atomic_primitive!(AtomicU64(u64), size("64"), align(8));
+-impl_atomic_primitive!(AtomicI128(i128), size("128"), align(16));
+-impl_atomic_primitive!(AtomicU128(u128), size("128"), align(16));
+ 
+ #[cfg(target_pointer_width = "16")]
+ impl_atomic_primitive!(AtomicIsize(isize), size("ptr"), align(2));
 @@ -3585,44 +3585,6 @@ pub const fn as_ptr(&self) -> *mut $int_type {
      8,
      u64 AtomicU64
diff --git a/compiler/rustc_codegen_cranelift/patches/0027-sysroot_tests-128bit-atomic-operations.patch b/compiler/rustc_codegen_cranelift/patches/0027-sysroot_tests-128bit-atomic-operations.patch
index 16c8488acdb..f6e6bbc2387 100644
--- a/compiler/rustc_codegen_cranelift/patches/0027-sysroot_tests-128bit-atomic-operations.patch
+++ b/compiler/rustc_codegen_cranelift/patches/0027-sysroot_tests-128bit-atomic-operations.patch
@@ -17,7 +17,7 @@ index 1e336bf..35e6f54 100644
 @@ -2,5 +2,4 @@
  // tidy-alphabetical-start
 -#![cfg_attr(target_has_atomic = "128", feature(integer_atomics))]
- #![cfg_attr(test, feature(cfg_match))]
+ #![cfg_attr(test, feature(cfg_select))]
  #![feature(alloc_layout_extra)]
  #![feature(array_chunks)]
 diff --git a/coretests/tests/atomic.rs b/coretests/tests/atomic.rs
diff --git a/compiler/rustc_codegen_cranelift/rust-toolchain b/compiler/rustc_codegen_cranelift/rust-toolchain
index ceff15b1180..af4bd6dc6b8 100644
--- a/compiler/rustc_codegen_cranelift/rust-toolchain
+++ b/compiler/rustc_codegen_cranelift/rust-toolchain
@@ -1,4 +1,4 @@
 [toolchain]
-channel = "nightly-2025-03-30"
+channel = "nightly-2025-05-25"
 components = ["rust-src", "rustc-dev", "llvm-tools"]
 profile = "minimal"
diff --git a/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh
index ca6426f2ba9..532702bb1a4 100644
--- a/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh
@@ -43,7 +43,7 @@ verbose-tests = false
 # disabled bootstrap will crash trying to copy llvm tools for the bootstrap
 # compiler.
 llvm-tools = false
-std-features = ["panic-unwind", "compiler-builtins-no-f16-f128"]
+std-features = ["panic-unwind"]
 
 EOF
 
diff --git a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
index 95a4302b5e4..32c71f433b0 100755
--- a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
@@ -53,6 +53,8 @@ rm -r tests/run-make/split-debuginfo # same
 rm -r tests/run-make/target-specs # i686 not supported by Cranelift
 rm -r tests/run-make/mismatching-target-triples # same
 rm tests/ui/asm/x86_64/issue-96797.rs # const and sym inline asm operands don't work entirely correctly
+rm tests/ui/asm/global-asm-mono-sym-fn.rs # same
+rm tests/ui/asm/naked-asm-mono-sym-fn.rs # same
 rm tests/ui/asm/x86_64/goto.rs # inline asm labels not supported
 rm tests/ui/simd/simd-bitmask-notpow2.rs # non-pow-of-2 simd vector sizes
 rm -r tests/run-make/embed-source-dwarf # embedding sources in debuginfo
@@ -70,19 +72,13 @@ rm tests/ui/consts/precise-drop-with-coverage.rs
 rm tests/ui/issues/issue-85461.rs
 rm -r tests/ui/instrument-coverage/
 
-# missing f16/f128 support
-rm tests/ui/half-open-range-patterns/half-open-range-pats-semantics.rs
-rm tests/ui/asm/aarch64/type-f16.rs
-rm tests/ui/float/conv-bits-runtime-const.rs
-rm tests/ui/consts/const-eval/float_methods.rs
-rm tests/ui/match/match-float.rs
-
 # optimization tests
 # ==================
 rm tests/ui/codegen/issue-28950.rs # depends on stack size optimizations
 rm tests/ui/codegen/init-large-type.rs # same
 rm -r tests/run-make/fmt-write-bloat/ # tests an optimization
 rm tests/ui/statics/const_generics.rs # same
+rm tests/ui/linking/executable-no-mangle-strip.rs # requires --gc-sections to work for statics
 
 # backend specific tests
 # ======================
@@ -96,6 +92,7 @@ rm -r tests/run-make/llvm-location-discriminator-limit-dummy-span # same
 rm tests/ui/abi/stack-protector.rs # requires stack protector support
 rm -r tests/run-make/emit-stack-sizes # requires support for -Z emit-stack-sizes
 rm -r tests/run-make/optimization-remarks-dir # remarks are LLVM specific
+rm -r tests/ui/optimization-remark.rs # same
 rm -r tests/run-make/print-to-output # requires --print relocation-models
 
 # requires asm, llvm-ir and/or llvm-bc emit support
@@ -118,7 +115,6 @@ rm tests/ui/mir/mir_raw_fat_ptr.rs # same
 rm tests/ui/consts/issue-33537.rs # same
 rm tests/ui/consts/const-mut-refs-crate.rs # same
 rm tests/ui/abi/large-byval-align.rs # exceeds implementation limit of Cranelift
-rm tests/ui/abi/simd-abi-checks-avx.rs # attempts to declare function with two different signatures
 
 # doesn't work due to the way the rustc test suite is invoked.
 # should work when using ./x.py test the way it is intended
@@ -129,7 +125,11 @@ rm -r tests/run-make/compiler-builtins # Expects lib/rustlib/src/rust to contain
 rm -r tests/run-make/translation # same
 rm -r tests/run-make/missing-unstable-trait-bound # This disables support for unstable features, but running cg_clif needs some unstable features
 rm -r tests/run-make/const-trait-stable-toolchain # same
+rm -r tests/run-make/print-request-help-stable-unstable # same
 rm -r tests/run-make/incr-add-rust-src-component
+rm tests/ui/errors/remap-path-prefix-sysroot.rs # different sysroot source path
+rm -r tests/run-make/export/extern-opt # something about rustc version mismatches
+rm -r tests/run-make/export # same
 
 # genuine bugs
 # ============
@@ -141,6 +141,7 @@ rm -r tests/run-make/panic-abort-eh_frame # .eh_frame emitted with panic=abort
 rm tests/ui/process/nofile-limit.rs # TODO some AArch64 linking issue
 rm tests/ui/backtrace/synchronized-panic-handler.rs # missing needs-unwind annotation
 rm tests/ui/lint/non-snake-case/lint-non-snake-case-crate.rs # same
+rm tests/ui/async-await/async-drop/async-drop-initial.rs # same (rust-lang/rust#140493)
 rm -r tests/ui/codegen/equal-pointers-unequal # make incorrect assumptions about the location of stack variables
 
 rm tests/ui/stdio-is-blocking.rs # really slow with unoptimized libstd
diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
index 5f631405a9a..fe5b220117f 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
@@ -10,7 +10,7 @@ use std::mem;
 use cranelift_codegen::ir::{ArgumentPurpose, SigRef};
 use cranelift_codegen::isa::CallConv;
 use cranelift_module::ModuleError;
-use rustc_abi::ExternAbi;
+use rustc_abi::{CanonAbi, ExternAbi, X86Call};
 use rustc_codegen_ssa::base::is_call_from_compiler_builtins_to_upstream_monomorphization;
 use rustc_codegen_ssa::errors::CompilerBuiltinsCannotCall;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
@@ -19,7 +19,8 @@ use rustc_middle::ty::layout::FnAbiOf;
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_session::Session;
 use rustc_span::source_map::Spanned;
-use rustc_target::callconv::{Conv, FnAbi, PassMode};
+use rustc_target::callconv::{FnAbi, PassMode};
+use smallvec::SmallVec;
 
 use self::pass_mode::*;
 pub(crate) use self::returning::codegen_return;
@@ -41,32 +42,27 @@ fn clif_sig_from_fn_abi<'tcx>(
     Signature { params, returns, call_conv }
 }
 
-pub(crate) fn conv_to_call_conv(sess: &Session, c: Conv, default_call_conv: CallConv) -> CallConv {
+pub(crate) fn conv_to_call_conv(
+    sess: &Session,
+    c: CanonAbi,
+    default_call_conv: CallConv,
+) -> CallConv {
     match c {
-        Conv::Rust | Conv::C => default_call_conv,
-        Conv::Cold | Conv::PreserveMost | Conv::PreserveAll => CallConv::Cold,
-        Conv::X86_64SysV => CallConv::SystemV,
-        Conv::X86_64Win64 => CallConv::WindowsFastcall,
-
-        // Should already get a back compat warning
-        Conv::X86Fastcall | Conv::X86Stdcall | Conv::X86ThisCall | Conv::X86VectorCall => {
-            default_call_conv
-        }
-
-        Conv::X86Intr | Conv::RiscvInterrupt { .. } => {
-            sess.dcx().fatal(format!("interrupt call conv {c:?} not yet implemented"))
-        }
-
-        Conv::ArmAapcs => sess.dcx().fatal("aapcs call conv not yet implemented"),
-        Conv::CCmseNonSecureCall => {
-            sess.dcx().fatal("C-cmse-nonsecure-call call conv is not yet implemented");
-        }
-        Conv::CCmseNonSecureEntry => {
-            sess.dcx().fatal("C-cmse-nonsecure-entry call conv is not yet implemented");
+        CanonAbi::Rust | CanonAbi::C => default_call_conv,
+        CanonAbi::RustCold => CallConv::Cold,
+
+        CanonAbi::X86(x86_call) => match x86_call {
+            X86Call::SysV64 => CallConv::SystemV,
+            X86Call::Win64 => CallConv::WindowsFastcall,
+            // Should already get a back compat warning
+            _ => default_call_conv,
+        },
+
+        CanonAbi::Interrupt(_) | CanonAbi::Arm(_) => {
+            sess.dcx().fatal("call conv {c:?} is not yet implemented")
         }
-
-        Conv::Msp430Intr | Conv::GpuKernel | Conv::AvrInterrupt | Conv::AvrNonBlockingInterrupt => {
-            unreachable!("tried to use {c:?} call conv which only exists on an unsupported target");
+        CanonAbi::GpuKernel => {
+            unreachable!("tried to use {c:?} call conv which only exists on an unsupported target")
         }
     }
 }
@@ -153,10 +149,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
 
             let ret = self.lib_call_unadjusted(name, params, returns, &args)[0];
 
-            // FIXME(bytecodealliance/wasmtime#6104) use bitcast instead of store to get from i64x2 to i128
-            let ret_ptr = self.create_stack_slot(16, 16);
-            ret_ptr.store(self, ret, MemFlags::trusted());
-            Cow::Owned(vec![ret_ptr.load(self, types::I128, MemFlags::trusted())])
+            Cow::Owned(vec![codegen_bitcast(self, types::I128, ret)])
         } else if ret_single_i128 && self.tcx.sess.target.arch == "s390x" {
             // Return i128 using a return area pointer on s390x.
             let mut params = params;
@@ -184,11 +177,9 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         let sig = Signature { params, returns, call_conv: self.target_config.default_call_conv };
         let func_id = self.module.declare_function(name, Linkage::Import, &sig).unwrap();
         let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func);
-        if self.clif_comments.enabled() {
-            self.add_comment(func_ref, format!("{:?}", name));
-        }
         let call_inst = self.bcx.ins().call(func_ref, args);
         if self.clif_comments.enabled() {
+            self.add_comment(func_ref, format!("{:?}", name));
             self.add_comment(call_inst, format!("lib_call {}", name));
         }
         let results = self.bcx.inst_results(call_inst);
@@ -384,6 +375,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
     args: &[Spanned<Operand<'tcx>>],
     destination: Place<'tcx>,
     target: Option<BasicBlock>,
+    _unwind: UnwindAction,
 ) {
     let func = codegen_operand(fx, func);
     let fn_sig = func.layout().ty.fn_sig(fx.tcx);
@@ -529,7 +521,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
         Some(Instance { def: InstanceKind::Virtual(_, idx), .. }) => {
             if fx.clif_comments.enabled() {
                 let nop_inst = fx.bcx.ins().nop();
-                fx.add_comment(
+                fx.add_post_comment(
                     nop_inst,
                     with_no_trimmed_paths!(format!(
                         "virtual call; self arg pass mode: {:?}",
@@ -555,7 +547,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
         None => {
             if fx.clif_comments.enabled() {
                 let nop_inst = fx.bcx.ins().nop();
-                fx.add_comment(nop_inst, "indirect call");
+                fx.add_post_comment(nop_inst, "indirect call");
             }
 
             let func = func.load_scalar(fx);
@@ -585,17 +577,18 @@ pub(crate) fn codegen_terminator_call<'tcx>(
             adjust_call_for_c_variadic(fx, &fn_abi, source_info, func_ref, &mut call_args);
         }
 
-        if fx.clif_comments.enabled() {
-            let nop_inst = fx.bcx.ins().nop();
-            with_no_trimmed_paths!(fx.add_comment(nop_inst, format!("abi: {:?}", fn_abi)));
-        }
-
-        match func_ref {
+        let call_inst = match func_ref {
             CallTarget::Direct(func_ref) => fx.bcx.ins().call(func_ref, &call_args),
             CallTarget::Indirect(sig, func_ptr) => {
                 fx.bcx.ins().call_indirect(sig, func_ptr, &call_args)
             }
+        };
+
+        if fx.clif_comments.enabled() {
+            with_no_trimmed_paths!(fx.add_comment(call_inst, format!("abi: {:?}", fn_abi)));
         }
+
+        fx.bcx.func.dfg.inst_results(call_inst).iter().copied().collect::<SmallVec<[Value; 2]>>()
     });
 
     if let Some(dest) = target {
@@ -612,7 +605,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
         target: CallTarget,
         call_args: &mut Vec<Value>,
     ) {
-        if fn_abi.conv != Conv::C {
+        if fn_abi.conv != CanonAbi::C {
             fx.tcx.dcx().span_fatal(
                 source_info.span,
                 format!("Variadic call for non-C abi {:?}", fn_abi.conv),
@@ -705,13 +698,16 @@ pub(crate) fn codegen_drop<'tcx>(
     source_info: mir::SourceInfo,
     drop_place: CPlace<'tcx>,
     target: BasicBlock,
+    _unwind: UnwindAction,
 ) {
     let ty = drop_place.layout().ty;
     let drop_instance = Instance::resolve_drop_in_place(fx.tcx, ty);
+    let ret_block = fx.get_block(target);
 
     // AsyncDropGlueCtorShim can't be here
     if let ty::InstanceKind::DropGlue(_, None) = drop_instance.def {
         // we don't actually need to drop anything
+        fx.bcx.ins().jump(ret_block, &[]);
     } else {
         match ty.kind() {
             ty::Dynamic(_, _, ty::Dyn) => {
@@ -748,7 +744,9 @@ pub(crate) fn codegen_drop<'tcx>(
 
                 let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi);
                 let sig = fx.bcx.import_signature(sig);
+                // FIXME implement cleanup on exceptions
                 fx.bcx.ins().call_indirect(sig, drop_fn, &[ptr]);
+                fx.bcx.ins().jump(ret_block, &[]);
             }
             ty::Dynamic(_, _, ty::DynStar) => {
                 // IN THIS ARM, WE HAVE:
@@ -792,6 +790,8 @@ pub(crate) fn codegen_drop<'tcx>(
                 let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi);
                 let sig = fx.bcx.import_signature(sig);
                 fx.bcx.ins().call_indirect(sig, drop_fn, &[data]);
+                // FIXME implement cleanup on exceptions
+                fx.bcx.ins().jump(ret_block, &[]);
             }
             _ => {
                 assert!(!matches!(drop_instance.def, InstanceKind::Virtual(_, _)));
@@ -817,10 +817,37 @@ pub(crate) fn codegen_drop<'tcx>(
 
                 let func_ref = fx.get_function_ref(drop_instance);
                 fx.bcx.ins().call(func_ref, &call_args);
+                // FIXME implement cleanup on exceptions
+                fx.bcx.ins().jump(ret_block, &[]);
             }
         }
     }
+}
 
-    let target_block = fx.get_block(target);
-    fx.bcx.ins().jump(target_block, &[]);
+pub(crate) fn lib_call_arg_param(tcx: TyCtxt<'_>, ty: Type, is_signed: bool) -> AbiParam {
+    let param = AbiParam::new(ty);
+    if ty.is_int() && u64::from(ty.bits()) < tcx.data_layout.pointer_size.bits() {
+        match (&*tcx.sess.target.arch, &*tcx.sess.target.vendor) {
+            ("x86_64", _) | ("aarch64", "apple") => match (ty, is_signed) {
+                (types::I8 | types::I16, true) => param.sext(),
+                (types::I8 | types::I16, false) => param.uext(),
+                _ => param,
+            },
+            ("aarch64", _) => param,
+            ("riscv64", _) => match (ty, is_signed) {
+                (types::I32, _) | (_, true) => param.sext(),
+                _ => param.uext(),
+            },
+            ("s390x", _) => {
+                if is_signed {
+                    param.sext()
+                } else {
+                    param.uext()
+                }
+            }
+            _ => unimplemented!("{:?}", tcx.sess.target.arch),
+        }
+    } else {
+        param
+    }
 }
diff --git a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
index 06d89bc9ea7..6d8614aca69 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
@@ -22,8 +22,10 @@ fn reg_to_abi_param(reg: Reg) -> AbiParam {
         (RegKind::Integer, 3..=4) => types::I32,
         (RegKind::Integer, 5..=8) => types::I64,
         (RegKind::Integer, 9..=16) => types::I128,
+        (RegKind::Float, 2) => types::F16,
         (RegKind::Float, 4) => types::F32,
         (RegKind::Float, 8) => types::F64,
+        (RegKind::Float, 16) => types::F128,
         (RegKind::Vector, size) => types::I8.by(u32::try_from(size).unwrap()).unwrap(),
         _ => unreachable!("{:?}", reg),
     };
diff --git a/compiler/rustc_codegen_cranelift/src/abi/returning.rs b/compiler/rustc_codegen_cranelift/src/abi/returning.rs
index 9e048c7badb..36087f96dd7 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/returning.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/returning.rs
@@ -46,7 +46,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     ret_arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
     ret_place: CPlace<'tcx>,
-    f: impl FnOnce(&mut FunctionCx<'_, '_, 'tcx>, Option<Value>) -> Inst,
+    f: impl FnOnce(&mut FunctionCx<'_, '_, 'tcx>, Option<Value>) -> SmallVec<[Value; 2]>,
 ) {
     let (ret_temp_place, return_ptr) = match ret_arg_abi.mode {
         PassMode::Ignore => (None, None),
@@ -67,23 +67,21 @@ pub(super) fn codegen_with_call_return_arg<'tcx>(
         PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast { .. } => (None, None),
     };
 
-    let call_inst = f(fx, return_ptr);
+    let results = f(fx, return_ptr);
 
     match ret_arg_abi.mode {
         PassMode::Ignore => {}
         PassMode::Direct(_) => {
-            let ret_val = fx.bcx.inst_results(call_inst)[0];
+            let ret_val = results[0];
             ret_place.write_cvalue(fx, CValue::by_val(ret_val, ret_arg_abi.layout));
         }
         PassMode::Pair(_, _) => {
-            let ret_val_a = fx.bcx.inst_results(call_inst)[0];
-            let ret_val_b = fx.bcx.inst_results(call_inst)[1];
+            let ret_val_a = results[0];
+            let ret_val_b = results[1];
             ret_place
                 .write_cvalue(fx, CValue::by_val_pair(ret_val_a, ret_val_b, ret_arg_abi.layout));
         }
         PassMode::Cast { ref cast, .. } => {
-            let results =
-                fx.bcx.inst_results(call_inst).iter().copied().collect::<SmallVec<[Value; 2]>>();
             let result =
                 super::pass_mode::from_casted_value(fx, &results, ret_place.layout(), cast);
             ret_place.write_cvalue(fx, result);
diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs
index 524e0d9fe35..0b641ba64b7 100644
--- a/compiler/rustc_codegen_cranelift/src/base.rs
+++ b/compiler/rustc_codegen_cranelift/src/base.rs
@@ -15,9 +15,9 @@ use rustc_middle::ty::print::with_no_trimmed_paths;
 
 use crate::constant::ConstantCx;
 use crate::debuginfo::{FunctionDebugContext, TypeDebugContext};
-use crate::enable_verifier;
 use crate::prelude::*;
 use crate::pretty_clif::CommentWriter;
+use crate::{codegen_f16_f128, enable_verifier};
 
 pub(crate) struct CodegenedFunction {
     symbol_name: String,
@@ -193,6 +193,18 @@ pub(crate) fn compile_fn(
                     name = codegened_func.symbol_name
                 ));
             }
+            Err(ModuleError::Compilation(CodegenError::Verifier(err))) => {
+                let early_dcx = rustc_session::EarlyDiagCtxt::new(
+                    rustc_session::config::ErrorOutputType::default(),
+                );
+                let _ = early_dcx.early_err(format!("{:?}", err));
+                let pretty_error = cranelift_codegen::print_errors::pretty_verifier_error(
+                    &context.func,
+                    Some(Box::new(&clif_comments)),
+                    err,
+                );
+                early_dcx.early_fatal(format!("cranelift verify error:\n{}", pretty_error));
+            }
             Err(err) => {
                 panic!("Error while defining {name}: {err:?}", name = codegened_func.symbol_name);
             }
@@ -303,7 +315,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                 bb_data.terminator().kind.fmt_head(&mut terminator_head).unwrap();
             });
             let inst = fx.bcx.func.layout.last_inst(block).unwrap();
-            fx.add_comment(inst, terminator_head);
+            fx.add_post_comment(inst, terminator_head);
         }
 
         let source_info = bb_data.terminator().source_info;
@@ -337,7 +349,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
             TerminatorKind::Return => {
                 crate::abi::codegen_return(fx);
             }
-            TerminatorKind::Assert { cond, expected, msg, target, unwind: _ } => {
+            TerminatorKind::Assert { cond, expected, msg, target, unwind } => {
                 if !fx.tcx.sess.overflow_checks() && msg.is_optional_overflow_check() {
                     let target = fx.get_block(*target);
                     fx.bcx.ins().jump(target, &[]);
@@ -367,7 +379,8 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                             fx,
                             rustc_hir::LangItem::PanicBoundsCheck,
                             &[index, len, location],
-                            Some(source_info.span),
+                            *unwind,
+                            source_info.span,
                         );
                     }
                     AssertKind::MisalignedPointerDereference { ref required, ref found } => {
@@ -379,7 +392,8 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                             fx,
                             rustc_hir::LangItem::PanicMisalignedPointerDereference,
                             &[required, found, location],
-                            Some(source_info.span),
+                            *unwind,
+                            source_info.span,
                         );
                     }
                     AssertKind::NullPointerDereference => {
@@ -389,7 +403,8 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                             fx,
                             rustc_hir::LangItem::PanicNullPointerDereference,
                             &[location],
-                            Some(source_info.span),
+                            *unwind,
+                            source_info.span,
                         )
                     }
                     _ => {
@@ -399,7 +414,8 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                             fx,
                             msg.panic_function(),
                             &[location],
-                            Some(source_info.span),
+                            *unwind,
+                            source_info.span,
                         );
                     }
                 }
@@ -457,7 +473,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                 destination,
                 target,
                 fn_span,
-                unwind: _,
+                unwind,
                 call_source: _,
             } => {
                 fx.tcx.prof.generic_activity("codegen call").run(|| {
@@ -468,6 +484,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                         args,
                         *destination,
                         *target,
+                        *unwind,
                     )
                 });
             }
@@ -514,7 +531,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                 );
             }
             TerminatorKind::UnwindTerminate(reason) => {
-                codegen_unwind_terminate(fx, source_info, *reason);
+                codegen_unwind_terminate(fx, source_info.span, *reason);
             }
             TerminatorKind::UnwindResume => {
                 // FIXME implement unwinding
@@ -530,23 +547,19 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
             | TerminatorKind::CoroutineDrop => {
                 bug!("shouldn't exist at codegen {:?}", bb_data.terminator());
             }
-            TerminatorKind::Drop { place, target, unwind: _, replace: _, drop, async_fut } => {
+            TerminatorKind::Drop { place, target, unwind, replace: _, drop, async_fut } => {
                 assert!(
                     async_fut.is_none() && drop.is_none(),
                     "Async Drop must be expanded or reset to sync before codegen"
                 );
                 let drop_place = codegen_place(fx, *place);
-                crate::abi::codegen_drop(fx, source_info, drop_place, *target);
+                crate::abi::codegen_drop(fx, source_info, drop_place, *target, *unwind);
             }
         };
     }
 }
 
-fn codegen_stmt<'tcx>(
-    fx: &mut FunctionCx<'_, '_, 'tcx>,
-    #[allow(unused_variables)] cur_block: Block,
-    stmt: &Statement<'tcx>,
-) {
+fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: &Statement<'tcx>) {
     let _print_guard = crate::PrintOnPanic(|| format!("stmt {:?}", stmt));
 
     fx.set_debug_loc(stmt.source_info);
@@ -557,7 +570,7 @@ fn codegen_stmt<'tcx>(
             if fx.clif_comments.enabled() {
                 let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap();
                 with_no_trimmed_paths!({
-                    fx.add_comment(inst, format!("{:?}", stmt));
+                    fx.add_post_comment(inst, format!("{:?}", stmt));
                 });
             }
         }
@@ -622,6 +635,15 @@ fn codegen_stmt<'tcx>(
                             let val = operand.load_scalar(fx);
                             match layout.ty.kind() {
                                 ty::Int(_) => CValue::by_val(fx.bcx.ins().ineg(val), layout),
+                                // FIXME(bytecodealliance/wasmtime#8312): Remove
+                                // once backend lowerings have been added to
+                                // Cranelift.
+                                ty::Float(FloatTy::F16) => {
+                                    CValue::by_val(codegen_f16_f128::neg_f16(fx, val), layout)
+                                }
+                                ty::Float(FloatTy::F128) => {
+                                    CValue::by_val(codegen_f16_f128::neg_f128(fx, val), layout)
+                                }
                                 ty::Float(_) => CValue::by_val(fx.bcx.ins().fneg(val), layout),
                                 _ => unreachable!("un op Neg for {:?}", layout.ty),
                             }
@@ -793,7 +815,7 @@ fn codegen_stmt<'tcx>(
                         let done_block = fx.bcx.create_block();
                         let index = fx.bcx.append_block_param(loop_block, fx.pointer_type);
                         let zero = fx.bcx.ins().iconst(fx.pointer_type, 0);
-                        fx.bcx.ins().jump(loop_block, &[zero]);
+                        fx.bcx.ins().jump(loop_block, &[zero.into()]);
 
                         fx.bcx.switch_to_block(loop_block);
                         let done = fx.bcx.ins().icmp_imm(IntCC::Equal, index, times as i64);
@@ -803,7 +825,7 @@ fn codegen_stmt<'tcx>(
                         let to = lval.place_index(fx, index);
                         to.write_cvalue(fx, operand);
                         let index = fx.bcx.ins().iadd_imm(index, 1);
-                        fx.bcx.ins().jump(loop_block, &[index]);
+                        fx.bcx.ins().jump(loop_block, &[index.into()]);
 
                         fx.bcx.switch_to_block(done_block);
                         fx.bcx.ins().nop();
@@ -1052,30 +1074,35 @@ pub(crate) fn codegen_operand<'tcx>(
 pub(crate) fn codegen_panic_nounwind<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     msg_str: &str,
-    span: Option<Span>,
+    span: Span,
 ) {
     let msg_ptr = fx.anonymous_str(msg_str);
     let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap());
     let args = [msg_ptr, msg_len];
 
-    codegen_panic_inner(fx, rustc_hir::LangItem::PanicNounwind, &args, span);
+    codegen_panic_inner(
+        fx,
+        rustc_hir::LangItem::PanicNounwind,
+        &args,
+        UnwindAction::Terminate(UnwindTerminateReason::Abi),
+        span,
+    );
 }
 
 pub(crate) fn codegen_unwind_terminate<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
-    source_info: mir::SourceInfo,
+    span: Span,
     reason: UnwindTerminateReason,
 ) {
-    let args = [];
-
-    codegen_panic_inner(fx, reason.lang_item(), &args, Some(source_info.span));
+    codegen_panic_inner(fx, reason.lang_item(), &[], UnwindAction::Unreachable, span);
 }
 
 fn codegen_panic_inner<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     lang_item: rustc_hir::LangItem,
     args: &[Value],
-    span: Option<Span>,
+    _unwind: UnwindAction,
+    span: Span,
 ) {
     fx.bcx.set_cold_block(fx.bcx.current_block().unwrap());
 
@@ -1090,6 +1117,8 @@ fn codegen_panic_inner<'tcx>(
 
     let symbol_name = fx.tcx.symbol_name(instance).name;
 
+    // FIXME implement cleanup on exceptions
+
     fx.lib_call(
         symbol_name,
         args.iter().map(|&arg| AbiParam::new(fx.bcx.func.dfg.value_type(arg))).collect(),
diff --git a/compiler/rustc_codegen_cranelift/src/cast.rs b/compiler/rustc_codegen_cranelift/src/cast.rs
index e2346324232..8a725680e70 100644
--- a/compiler/rustc_codegen_cranelift/src/cast.rs
+++ b/compiler/rustc_codegen_cranelift/src/cast.rs
@@ -1,5 +1,6 @@
 //! Various number casting functions
 
+use crate::codegen_f16_f128;
 use crate::prelude::*;
 
 pub(crate) fn clif_intcast(
@@ -36,6 +37,14 @@ pub(crate) fn clif_int_or_float_cast(
 ) -> Value {
     let from_ty = fx.bcx.func.dfg.value_type(from);
 
+    // FIXME(bytecodealliance/wasmtime#8312): Remove in favour of native
+    // Cranelift operations once Cranelift backends have lowerings for them.
+    if matches!(from_ty, types::F16 | types::F128)
+        || matches!(to_ty, types::F16 | types::F128) && from_ty != to_ty
+    {
+        return codegen_f16_f128::codegen_cast(fx, from, from_signed, to_ty, to_signed);
+    }
+
     if from_ty.is_int() && to_ty.is_int() {
         // int-like -> int-like
         clif_intcast(
@@ -58,8 +67,10 @@ pub(crate) fn clif_int_or_float_cast(
                 "__float{sign}ti{flt}f",
                 sign = if from_signed { "" } else { "un" },
                 flt = match to_ty {
+                    types::F16 => "h",
                     types::F32 => "s",
                     types::F64 => "d",
+                    types::F128 => "t",
                     _ => unreachable!("{:?}", to_ty),
                 },
             );
@@ -90,8 +101,10 @@ pub(crate) fn clif_int_or_float_cast(
                 "__fix{sign}{flt}fti",
                 sign = if to_signed { "" } else { "uns" },
                 flt = match from_ty {
+                    types::F16 => "h",
                     types::F32 => "s",
                     types::F64 => "d",
+                    types::F128 => "t",
                     _ => unreachable!("{:?}", to_ty),
                 },
             );
@@ -145,8 +158,12 @@ pub(crate) fn clif_int_or_float_cast(
     } else if from_ty.is_float() && to_ty.is_float() {
         // float -> float
         match (from_ty, to_ty) {
-            (types::F32, types::F64) => fx.bcx.ins().fpromote(types::F64, from),
-            (types::F64, types::F32) => fx.bcx.ins().fdemote(types::F32, from),
+            (types::F16, types::F32 | types::F64 | types::F128)
+            | (types::F32, types::F64 | types::F128)
+            | (types::F64, types::F128) => fx.bcx.ins().fpromote(to_ty, from),
+            (types::F128, types::F64 | types::F32 | types::F16)
+            | (types::F64, types::F32 | types::F16)
+            | (types::F32, types::F16) => fx.bcx.ins().fdemote(to_ty, from),
             _ => from,
         }
     } else {
diff --git a/compiler/rustc_codegen_cranelift/src/codegen_f16_f128.rs b/compiler/rustc_codegen_cranelift/src/codegen_f16_f128.rs
new file mode 100644
index 00000000000..1e202be1f18
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/src/codegen_f16_f128.rs
@@ -0,0 +1,284 @@
+use crate::prelude::*;
+
+pub(crate) fn f16_to_f32(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
+    let (value, arg_ty) =
+        if fx.tcx.sess.target.vendor == "apple" && fx.tcx.sess.target.arch == "x86_64" {
+            (
+                fx.bcx.ins().bitcast(types::I16, MemFlags::new(), value),
+                lib_call_arg_param(fx.tcx, types::I16, false),
+            )
+        } else {
+            (value, AbiParam::new(types::F16))
+        };
+    fx.lib_call("__extendhfsf2", vec![arg_ty], vec![AbiParam::new(types::F32)], &[value])[0]
+}
+
+fn f16_to_f64(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
+    let ret = f16_to_f32(fx, value);
+    fx.bcx.ins().fpromote(types::F64, ret)
+}
+
+pub(crate) fn f32_to_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
+    let ret_ty = if fx.tcx.sess.target.vendor == "apple" && fx.tcx.sess.target.arch == "x86_64" {
+        types::I16
+    } else {
+        types::F16
+    };
+    let ret = fx.lib_call(
+        "__truncsfhf2",
+        vec![AbiParam::new(types::F32)],
+        vec![AbiParam::new(ret_ty)],
+        &[value],
+    )[0];
+    if ret_ty == types::I16 { fx.bcx.ins().bitcast(types::F16, MemFlags::new(), ret) } else { ret }
+}
+
+fn f64_to_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
+    let ret_ty = if fx.tcx.sess.target.vendor == "apple" && fx.tcx.sess.target.arch == "x86_64" {
+        types::I16
+    } else {
+        types::F16
+    };
+    let ret = fx.lib_call(
+        "__truncdfhf2",
+        vec![AbiParam::new(types::F64)],
+        vec![AbiParam::new(ret_ty)],
+        &[value],
+    )[0];
+    if ret_ty == types::I16 { fx.bcx.ins().bitcast(types::F16, MemFlags::new(), ret) } else { ret }
+}
+
+pub(crate) fn fcmp(fx: &mut FunctionCx<'_, '_, '_>, cc: FloatCC, lhs: Value, rhs: Value) -> Value {
+    let ty = fx.bcx.func.dfg.value_type(lhs);
+    match ty {
+        types::F32 | types::F64 => fx.bcx.ins().fcmp(cc, lhs, rhs),
+        types::F16 => {
+            let lhs = f16_to_f32(fx, lhs);
+            let rhs = f16_to_f32(fx, rhs);
+            fx.bcx.ins().fcmp(cc, lhs, rhs)
+        }
+        types::F128 => {
+            let (name, int_cc) = match cc {
+                FloatCC::Equal => ("__eqtf2", IntCC::Equal),
+                FloatCC::NotEqual => ("__netf2", IntCC::NotEqual),
+                FloatCC::LessThan => ("__lttf2", IntCC::SignedLessThan),
+                FloatCC::LessThanOrEqual => ("__letf2", IntCC::SignedLessThanOrEqual),
+                FloatCC::GreaterThan => ("__gttf2", IntCC::SignedGreaterThan),
+                FloatCC::GreaterThanOrEqual => ("__getf2", IntCC::SignedGreaterThanOrEqual),
+                _ => unreachable!("not currently used in rustc_codegen_cranelift: {cc:?}"),
+            };
+            let res = fx.lib_call(
+                name,
+                vec![AbiParam::new(types::F128), AbiParam::new(types::F128)],
+                // FIXME(rust-lang/compiler-builtins#919): This should be `I64` on non-AArch64
+                // architectures, but switching it before compiler-builtins is fixed causes test
+                // failures.
+                vec![AbiParam::new(types::I32)],
+                &[lhs, rhs],
+            )[0];
+            let zero = fx.bcx.ins().iconst(types::I32, 0);
+            let res = fx.bcx.ins().icmp(int_cc, res, zero);
+            res
+        }
+        _ => unreachable!("{ty:?}"),
+    }
+}
+
+pub(crate) fn codegen_f128_binop(
+    fx: &mut FunctionCx<'_, '_, '_>,
+    bin_op: BinOp,
+    lhs: Value,
+    rhs: Value,
+) -> Value {
+    let name = match bin_op {
+        BinOp::Add => "__addtf3",
+        BinOp::Sub => "__subtf3",
+        BinOp::Mul => "__multf3",
+        BinOp::Div => "__divtf3",
+        _ => unreachable!("handled in `codegen_float_binop`"),
+    };
+    fx.lib_call(
+        name,
+        vec![AbiParam::new(types::F128), AbiParam::new(types::F128)],
+        vec![AbiParam::new(types::F128)],
+        &[lhs, rhs],
+    )[0]
+}
+
+pub(crate) fn neg_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
+    let bits = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), value);
+    let bits = fx.bcx.ins().bxor_imm(bits, 0x8000);
+    fx.bcx.ins().bitcast(types::F16, MemFlags::new(), bits)
+}
+
+pub(crate) fn neg_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
+    let bits = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), value);
+    let (low, high) = fx.bcx.ins().isplit(bits);
+    let high = fx.bcx.ins().bxor_imm(high, 0x8000_0000_0000_0000_u64 as i64);
+    let bits = fx.bcx.ins().iconcat(low, high);
+    fx.bcx.ins().bitcast(types::F128, MemFlags::new(), bits)
+}
+
+pub(crate) fn abs_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
+    let bits = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), value);
+    let bits = fx.bcx.ins().band_imm(bits, 0x7fff);
+    fx.bcx.ins().bitcast(types::F16, MemFlags::new(), bits)
+}
+
+pub(crate) fn abs_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
+    let bits = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), value);
+    let (low, high) = fx.bcx.ins().isplit(bits);
+    let high = fx.bcx.ins().band_imm(high, 0x7fff_ffff_ffff_ffff_u64 as i64);
+    let bits = fx.bcx.ins().iconcat(low, high);
+    fx.bcx.ins().bitcast(types::F128, MemFlags::new(), bits)
+}
+
+pub(crate) fn copysign_f16(fx: &mut FunctionCx<'_, '_, '_>, lhs: Value, rhs: Value) -> Value {
+    let lhs = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), lhs);
+    let rhs = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), rhs);
+    let res = fx.bcx.ins().band_imm(lhs, 0x7fff);
+    let sign = fx.bcx.ins().band_imm(rhs, 0x8000);
+    let res = fx.bcx.ins().bor(res, sign);
+    fx.bcx.ins().bitcast(types::F16, MemFlags::new(), res)
+}
+
+pub(crate) fn copysign_f128(fx: &mut FunctionCx<'_, '_, '_>, lhs: Value, rhs: Value) -> Value {
+    let lhs = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), lhs);
+    let rhs = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), rhs);
+    let (low, lhs_high) = fx.bcx.ins().isplit(lhs);
+    let (_, rhs_high) = fx.bcx.ins().isplit(rhs);
+    let high = fx.bcx.ins().band_imm(lhs_high, 0x7fff_ffff_ffff_ffff_u64 as i64);
+    let sign = fx.bcx.ins().band_imm(rhs_high, 0x8000_0000_0000_0000_u64 as i64);
+    let high = fx.bcx.ins().bor(high, sign);
+    let res = fx.bcx.ins().iconcat(low, high);
+    fx.bcx.ins().bitcast(types::F128, MemFlags::new(), res)
+}
+
+pub(crate) fn codegen_cast(
+    fx: &mut FunctionCx<'_, '_, '_>,
+    from: Value,
+    from_signed: bool,
+    to_ty: Type,
+    to_signed: bool,
+) -> Value {
+    let from_ty = fx.bcx.func.dfg.value_type(from);
+    if from_ty.is_float() && to_ty.is_float() {
+        let name = match (from_ty, to_ty) {
+            (types::F16, types::F32) => return f16_to_f32(fx, from),
+            (types::F16, types::F64) => return f16_to_f64(fx, from),
+            (types::F16, types::F128) => "__extendhftf2",
+            (types::F32, types::F128) => "__extendsftf2",
+            (types::F64, types::F128) => "__extenddftf2",
+            (types::F128, types::F64) => "__trunctfdf2",
+            (types::F128, types::F32) => "__trunctfsf2",
+            (types::F128, types::F16) => "__trunctfhf2",
+            (types::F64, types::F16) => return f64_to_f16(fx, from),
+            (types::F32, types::F16) => return f32_to_f16(fx, from),
+            _ => unreachable!("{from_ty:?} -> {to_ty:?}"),
+        };
+        fx.lib_call(name, vec![AbiParam::new(from_ty)], vec![AbiParam::new(to_ty)], &[from])[0]
+    } else if from_ty.is_int() && to_ty == types::F16 {
+        let res = clif_int_or_float_cast(fx, from, from_signed, types::F32, false);
+        f32_to_f16(fx, res)
+    } else if from_ty == types::F16 && to_ty.is_int() {
+        let from = f16_to_f32(fx, from);
+        clif_int_or_float_cast(fx, from, false, to_ty, to_signed)
+    } else if from_ty.is_int() && to_ty == types::F128 {
+        let (from, from_ty) = if from_ty.bits() < 32 {
+            (clif_int_or_float_cast(fx, from, from_signed, types::I32, from_signed), types::I32)
+        } else {
+            (from, from_ty)
+        };
+        let name = format!(
+            "__float{sign}{size}itf",
+            sign = if from_signed { "" } else { "un" },
+            size = match from_ty {
+                types::I32 => 's',
+                types::I64 => 'd',
+                types::I128 => 't',
+                _ => unreachable!("{from_ty:?}"),
+            },
+        );
+        fx.lib_call(
+            &name,
+            vec![lib_call_arg_param(fx.tcx, from_ty, from_signed)],
+            vec![AbiParam::new(to_ty)],
+            &[from],
+        )[0]
+    } else if from_ty == types::F128 && to_ty.is_int() {
+        let ret_ty = if to_ty.bits() < 32 { types::I32 } else { to_ty };
+        let name = format!(
+            "__fix{sign}tf{size}i",
+            sign = if from_signed { "" } else { "un" },
+            size = match ret_ty {
+                types::I32 => 's',
+                types::I64 => 'd',
+                types::I128 => 't',
+                _ => unreachable!("{from_ty:?}"),
+            },
+        );
+        let ret =
+            fx.lib_call(&name, vec![AbiParam::new(from_ty)], vec![AbiParam::new(to_ty)], &[from])
+                [0];
+        let val = if ret_ty == to_ty {
+            ret
+        } else {
+            let (min, max) = match (to_ty, to_signed) {
+                (types::I8, false) => (0, i64::from(u8::MAX)),
+                (types::I16, false) => (0, i64::from(u16::MAX)),
+                (types::I8, true) => (i64::from(i8::MIN as u32), i64::from(i8::MAX as u32)),
+                (types::I16, true) => (i64::from(i16::MIN as u32), i64::from(i16::MAX as u32)),
+                _ => unreachable!("{to_ty:?}"),
+            };
+            let min_val = fx.bcx.ins().iconst(types::I32, min);
+            let max_val = fx.bcx.ins().iconst(types::I32, max);
+
+            let val = if to_signed {
+                let has_underflow = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, ret, min);
+                let has_overflow = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThan, ret, max);
+                let bottom_capped = fx.bcx.ins().select(has_underflow, min_val, ret);
+                fx.bcx.ins().select(has_overflow, max_val, bottom_capped)
+            } else {
+                let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, ret, max);
+                fx.bcx.ins().select(has_overflow, max_val, ret)
+            };
+            fx.bcx.ins().ireduce(to_ty, val)
+        };
+
+        if let Some(false) = fx.tcx.sess.opts.unstable_opts.saturating_float_casts {
+            return val;
+        }
+
+        let is_not_nan = fcmp(fx, FloatCC::Equal, from, from);
+        let zero = type_zero_value(&mut fx.bcx, to_ty);
+        fx.bcx.ins().select(is_not_nan, val, zero)
+    } else {
+        unreachable!("{from_ty:?} -> {to_ty:?}");
+    }
+}
+
+pub(crate) fn fma_f16(fx: &mut FunctionCx<'_, '_, '_>, x: Value, y: Value, z: Value) -> Value {
+    let x = f16_to_f64(fx, x);
+    let y = f16_to_f64(fx, y);
+    let z = f16_to_f64(fx, z);
+    let res = fx.bcx.ins().fma(x, y, z);
+    f64_to_f16(fx, res)
+}
+
+pub(crate) fn fmin_f128(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value {
+    fx.lib_call(
+        "fminimumf128",
+        vec![AbiParam::new(types::F128), AbiParam::new(types::F128)],
+        vec![AbiParam::new(types::F128)],
+        &[a, b],
+    )[0]
+}
+
+pub(crate) fn fmax_f128(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value {
+    fx.lib_call(
+        "fmaximumf128",
+        vec![AbiParam::new(types::F128), AbiParam::new(types::F128)],
+        vec![AbiParam::new(types::F128)],
+        &[a, b],
+    )[0]
+}
diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs
index abe2972ba0c..2f11b2d2dcc 100644
--- a/compiler/rustc_codegen_cranelift/src/common.rs
+++ b/compiler/rustc_codegen_cranelift/src/common.rs
@@ -33,10 +33,10 @@ pub(crate) fn scalar_to_clif_type(tcx: TyCtxt<'_>, scalar: Scalar) -> Type {
             Integer::I128 => types::I128,
         },
         Primitive::Float(float) => match float {
-            Float::F16 => unimplemented!("f16_f128"),
+            Float::F16 => types::F16,
             Float::F32 => types::F32,
             Float::F64 => types::F64,
-            Float::F128 => unimplemented!("f16_f128"),
+            Float::F128 => types::F128,
         },
         // FIXME(erikdesjardins): handle non-default addrspace ptr sizes
         Primitive::Pointer(_) => pointer_ty(tcx),
@@ -64,10 +64,10 @@ fn clif_type_from_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option<types::Typ
         },
         ty::Char => types::I32,
         ty::Float(size) => match size {
-            FloatTy::F16 => unimplemented!("f16_f128"),
+            FloatTy::F16 => types::F16,
             FloatTy::F32 => types::F32,
             FloatTy::F64 => types::F64,
-            FloatTy::F128 => unimplemented!("f16_f128"),
+            FloatTy::F128 => types::F128,
         },
         ty::FnPtr(..) => pointer_ty(tcx),
         ty::RawPtr(pointee_ty, _) | ty::Ref(_, pointee_ty, _) => {
diff --git a/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs b/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs
index bf16e81a06f..6eea19211fa 100644
--- a/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs
+++ b/compiler/rustc_codegen_cranelift/src/compiler_builtins.rs
@@ -46,15 +46,100 @@ builtin_functions! {
     fn __rust_u128_mulo(a: u128, b: u128, oflow: &mut i32) -> u128;
     fn __rust_i128_mulo(a: i128, b: i128, oflow: &mut i32) -> i128;
 
-    // floats
+    // integer -> float
     fn __floattisf(i: i128) -> f32;
     fn __floattidf(i: i128) -> f64;
+    fn __floatsitf(i: i32) -> f128;
+    fn __floatditf(i: i64) -> f128;
+    fn __floattitf(i: i128) -> f128;
     fn __floatuntisf(i: u128) -> f32;
     fn __floatuntidf(i: u128) -> f64;
+    fn __floatunsitf(i: u32) -> f128;
+    fn __floatunditf(i: u64) -> f128;
+    fn __floatuntitf(i: u128) -> f128;
+    // float -> integer
     fn __fixsfti(f: f32) -> i128;
     fn __fixdfti(f: f64) -> i128;
+    fn __fixtfsi(f: f128) -> i32;
+    fn __fixtfdi(f: f128) -> i64;
+    fn __fixtfti(f: f128) -> i128;
     fn __fixunssfti(f: f32) -> u128;
     fn __fixunsdfti(f: f64) -> u128;
+    fn __fixunstfsi(f: f128) -> u32;
+    fn __fixunstfdi(f: f128) -> u64;
+    fn __fixunstfti(f: f128) -> u128;
+    // float -> float
+    fn __extendhfsf2(f: f16) -> f32;
+    fn __extendhftf2(f: f16) -> f128;
+    fn __extendsftf2(f: f32) -> f128;
+    fn __extenddftf2(f: f64) -> f128;
+    fn __trunctfdf2(f: f128) -> f64;
+    fn __trunctfsf2(f: f128) -> f32;
+    fn __trunctfhf2(f: f128) -> f16;
+    fn __truncdfhf2(f: f64) -> f16;
+    fn __truncsfhf2(f: f32) -> f16;
+    // float binops
+    fn __addtf3(a: f128, b: f128) -> f128;
+    fn __subtf3(a: f128, b: f128) -> f128;
+    fn __multf3(a: f128, b: f128) -> f128;
+    fn __divtf3(a: f128, b: f128) -> f128;
+    fn fmodf(a: f32, b: f32) -> f32;
+    fn fmod(a: f64, b: f64) -> f64;
+    fn fmodf128(a: f128, b: f128) -> f128;
+    // float comparison
+    fn __eqtf2(a: f128, b: f128) -> i32;
+    fn __netf2(a: f128, b: f128) -> i32;
+    fn __lttf2(a: f128, b: f128) -> i32;
+    fn __letf2(a: f128, b: f128) -> i32;
+    fn __gttf2(a: f128, b: f128) -> i32;
+    fn __getf2(a: f128, b: f128) -> i32;
+    fn fminimumf128(a: f128, b: f128) -> f128;
+    fn fmaximumf128(a: f128, b: f128) -> f128;
+    // Cranelift float libcalls
+    fn fmaf(a: f32, b: f32, c: f32) -> f32;
+    fn fma(a: f64, b: f64, c: f64) -> f64;
+    fn floorf(f: f32) -> f32;
+    fn floor(f: f64) -> f64;
+    fn ceilf(f: f32) -> f32;
+    fn ceil(f: f64) -> f64;
+    fn truncf(f: f32) -> f32;
+    fn trunc(f: f64) -> f64;
+    fn nearbyintf(f: f32) -> f32;
+    fn nearbyint(f: f64) -> f64;
+    // float intrinsics
+    fn __powisf2(a: f32, b: i32) -> f32;
+    fn __powidf2(a: f64, b: i32) -> f64;
+    // FIXME(f16_f128): `compiler-builtins` doesn't currently support `__powitf2` on MSVC.
+    // fn __powitf2(a: f128, b: i32) -> f128;
+    fn powf(a: f32, b: f32) -> f32;
+    fn pow(a: f64, b: f64) -> f64;
+    fn expf(f: f32) -> f32;
+    fn exp(f: f64) -> f64;
+    fn exp2f(f: f32) -> f32;
+    fn exp2(f: f64) -> f64;
+    fn logf(f: f32) -> f32;
+    fn log(f: f64) -> f64;
+    fn log2f(f: f32) -> f32;
+    fn log2(f: f64) -> f64;
+    fn log10f(f: f32) -> f32;
+    fn log10(f: f64) -> f64;
+    fn sinf(f: f32) -> f32;
+    fn sin(f: f64) -> f64;
+    fn cosf(f: f32) -> f32;
+    fn cos(f: f64) -> f64;
+    fn fmaf128(a: f128, b: f128, c: f128) -> f128;
+    fn floorf16(f: f16) -> f16;
+    fn floorf128(f: f128) -> f128;
+    fn ceilf16(f: f16) -> f16;
+    fn ceilf128(f: f128) -> f128;
+    fn truncf16(f: f16) -> f16;
+    fn truncf128(f: f128) -> f128;
+    fn rintf16(f: f16) -> f16;
+    fn rintf128(f: f128) -> f128;
+    fn sqrtf16(f: f16) -> f16;
+    fn sqrtf128(f: f128) -> f128;
+    // FIXME(f16_f128): Add other float intrinsics as compiler-builtins gains support (meaning they
+    // are available on all targets).
 
     // allocator
     // NOTE: These need to be mentioned here despite not being part of compiler_builtins because
@@ -67,5 +152,4 @@ builtin_functions! {
     fn malloc(size: size_t) -> *mut c_void;
     fn realloc(p: *mut c_void, size: size_t) -> *mut c_void;
     fn free(p: *mut c_void) -> ();
-
 }
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
index ccdc347af66..0f4696b9337 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
@@ -81,13 +81,36 @@ impl WriterRelocate {
     /// Perform the collected relocations to be usable for JIT usage.
     #[cfg(all(feature = "jit", not(windows)))]
     pub(super) fn relocate_for_jit(mut self, jit_module: &cranelift_jit::JITModule) -> Vec<u8> {
+        use cranelift_module::Module;
+
         for reloc in self.relocs.drain(..) {
             match reloc.name {
                 super::DebugRelocName::Section(_) => unreachable!(),
                 super::DebugRelocName::Symbol(sym) => {
-                    let addr = jit_module.get_finalized_function(
-                        cranelift_module::FuncId::from_u32(sym.try_into().unwrap()),
-                    );
+                    let addr = if sym & 1 << 31 == 0 {
+                        let func_id = FuncId::from_u32(sym.try_into().unwrap());
+                        // FIXME make JITModule::get_address public and use it here instead.
+                        // HACK rust_eh_personality is likely not defined in the same crate,
+                        // so get_finalized_function won't work. Use the rust_eh_personality
+                        // of cg_clif itself, which is likely ABI compatible.
+                        if jit_module.declarations().get_function_decl(func_id).name.as_deref()
+                            == Some("rust_eh_personality")
+                        {
+                            extern "C" {
+                                fn rust_eh_personality() -> !;
+                            }
+                            rust_eh_personality as *const u8
+                        } else {
+                            jit_module.get_finalized_function(func_id)
+                        }
+                    } else {
+                        jit_module
+                            .get_finalized_data(DataId::from_u32(
+                                u32::try_from(sym).unwrap() & !(1 << 31),
+                            ))
+                            .0
+                    };
+
                     let val = (addr as u64 as i64 + reloc.addend) as u64;
                     self.writer.write_udata_at(reloc.offset as usize, val, reloc.size).unwrap();
                 }
@@ -196,6 +219,16 @@ impl Writer for WriterRelocate {
                     });
                     self.write_udata(0, size)
                 }
+                gimli::DW_EH_PE_absptr => {
+                    self.relocs.push(DebugReloc {
+                        offset: self.len() as u32,
+                        size: size.into(),
+                        name: DebugRelocName::Symbol(symbol),
+                        addend,
+                        kind: object::RelocationKind::Absolute,
+                    });
+                    self.write_udata(0, size.into())
+                }
                 _ => Err(gimli::write::Error::UnsupportedPointerEncoding(eh_pe)),
             },
         }
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs
index 362333d35a4..74b82a7139a 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs
@@ -1,7 +1,6 @@
 //! Unwind info generation (`.eh_frame`)
 
 use cranelift_codegen::ir::Endianness;
-use cranelift_codegen::isa::TargetIsa;
 use cranelift_codegen::isa::unwind::UnwindInfo;
 use cranelift_object::ObjectProduct;
 use gimli::RunTimeEndian;
@@ -18,14 +17,14 @@ pub(crate) struct UnwindContext {
 }
 
 impl UnwindContext {
-    pub(crate) fn new(isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self {
-        let endian = match isa.endianness() {
+    pub(crate) fn new(module: &mut dyn Module, pic_eh_frame: bool) -> Self {
+        let endian = match module.isa().endianness() {
             Endianness::Little => RunTimeEndian::Little,
             Endianness::Big => RunTimeEndian::Big,
         };
         let mut frame_table = FrameTable::default();
 
-        let cie_id = if let Some(mut cie) = isa.create_systemv_cie() {
+        let cie_id = if let Some(mut cie) = module.isa().create_systemv_cie() {
             if pic_eh_frame {
                 cie.fde_address_encoding =
                     gimli::DwEhPe(gimli::DW_EH_PE_pcrel.0 | gimli::DW_EH_PE_sdata4.0);
@@ -38,8 +37,15 @@ impl UnwindContext {
         UnwindContext { endian, frame_table, cie_id }
     }
 
-    pub(crate) fn add_function(&mut self, func_id: FuncId, context: &Context, isa: &dyn TargetIsa) {
-        if let target_lexicon::OperatingSystem::MacOSX { .. } = isa.triple().operating_system {
+    pub(crate) fn add_function(
+        &mut self,
+        module: &mut dyn Module,
+        func_id: FuncId,
+        context: &Context,
+    ) {
+        if let target_lexicon::OperatingSystem::MacOSX { .. } =
+            module.isa().triple().operating_system
+        {
             // The object crate doesn't currently support DW_GNU_EH_PE_absptr, which macOS
             // requires for unwinding tables. In addition on arm64 it currently doesn't
             // support 32bit relocations as we currently use for the unwinding table.
@@ -48,7 +54,7 @@ impl UnwindContext {
         }
 
         let unwind_info = if let Some(unwind_info) =
-            context.compiled_code().unwrap().create_unwind_info(isa).unwrap()
+            context.compiled_code().unwrap().create_unwind_info(module.isa()).unwrap()
         {
             unwind_info
         } else {
diff --git a/compiler/rustc_codegen_cranelift/src/discriminant.rs b/compiler/rustc_codegen_cranelift/src/discriminant.rs
index 4d0d5dc60eb..a08b0e0cbfc 100644
--- a/compiler/rustc_codegen_cranelift/src/discriminant.rs
+++ b/compiler/rustc_codegen_cranelift/src/discriminant.rs
@@ -28,7 +28,7 @@ pub(crate) fn codegen_set_discriminant<'tcx>(
             tag_encoding: TagEncoding::Direct,
             variants: _,
         } => {
-            let ptr = place.place_field(fx, FieldIdx::new(tag_field));
+            let ptr = place.place_field(fx, tag_field);
             let to = layout.ty.discriminant_for_variant(fx.tcx, variant_index).unwrap().val;
             let to = match ptr.layout().ty.kind() {
                 ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
@@ -53,7 +53,7 @@ pub(crate) fn codegen_set_discriminant<'tcx>(
             variants: _,
         } => {
             if variant_index != untagged_variant {
-                let niche = place.place_field(fx, FieldIdx::new(tag_field));
+                let niche = place.place_field(fx, tag_field);
                 let niche_type = fx.clif_type(niche.layout().ty).unwrap();
                 let niche_value = variant_index.as_u32() - niche_variants.start().as_u32();
                 let niche_value = (niche_value as u128).wrapping_add(niche_start);
@@ -118,7 +118,7 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
     let cast_to = fx.clif_type(dest_layout.ty).unwrap();
 
     // Read the tag/niche-encoded discriminant from memory.
-    let tag = value.value_field(fx, FieldIdx::new(tag_field));
+    let tag = value.value_field(fx, tag_field);
     let tag = tag.load_scalar(fx);
 
     // Decode the discriminant (specifically if it's niche-encoded).
diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
index e368cf4386d..b1f185b551c 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
@@ -6,6 +6,7 @@ use std::os::raw::{c_char, c_int};
 
 use cranelift_jit::{JITBuilder, JITModule};
 use rustc_codegen_ssa::CrateInfo;
+use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::mono::MonoItem;
 use rustc_session::Session;
 use rustc_span::sym;
@@ -84,7 +85,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, jit_args: Vec<String>) -> ! {
 
     tcx.dcx().abort_if_errors();
 
-    jit_module.finalize_definitions();
+    let mut jit_module = jit_module.finalize_definitions();
 
     println!(
         "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed"
@@ -104,7 +105,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, jit_args: Vec<String>) -> ! {
         call_conv: jit_module.target_config().default_call_conv,
     };
     let start_func_id = jit_module.declare_function("main", Linkage::Import, &start_sig).unwrap();
-    let finalized_start: *const u8 = jit_module.module.get_finalized_function(start_func_id);
+    let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id);
 
     let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
         unsafe { ::std::mem::transmute(finalized_start) };
@@ -119,7 +120,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, jit_args: Vec<String>) -> ! {
     std::process::exit(ret);
 }
 
-pub(crate) fn codegen_and_compile_fn<'tcx>(
+fn codegen_and_compile_fn<'tcx>(
     tcx: TyCtxt<'tcx>,
     cx: &mut crate::CodegenCx,
     cached_context: &mut Context,
diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
index afee5095549..120d6ff9e38 100644
--- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs
+++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
@@ -850,7 +850,7 @@ fn asm_clif_type<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> Option<ty
         // Adapted from https://github.com/rust-lang/rust/blob/f3c66088610c1b80110297c2d9a8b5f9265b013f/compiler/rustc_hir_analysis/src/check/intrinsicck.rs#L136-L151
         ty::Adt(adt, args) if fx.tcx.is_lang_item(adt.did(), LangItem::MaybeUninit) => {
             let fields = &adt.non_enum_variant().fields;
-            let ty = fields[FieldIdx::from_u32(1)].ty(fx.tcx, args);
+            let ty = fields[FieldIdx::ONE].ty(fx.tcx, args);
             let ty::Adt(ty, args) = ty.kind() else {
                 unreachable!("expected first field of `MaybeUninit` to be an ADT")
             };
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
index eb0dfbb69c3..2dee9176936 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
@@ -62,11 +62,24 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
             });
         }
 
+        _ if intrinsic.starts_with("llvm.roundeven.v") => {
+            intrinsic_args!(fx, args => (v); intrinsic);
+
+            simd_for_each_lane(fx, v, ret, &|fx, _lane_ty, _res_lane_ty, lane| {
+                fx.bcx.ins().nearest(lane)
+            });
+        }
+
         _ => {
             fx.tcx
                 .dcx()
                 .warn(format!("unsupported llvm intrinsic {}; replacing with trap", intrinsic));
-            crate::trap::trap_unimplemented(fx, intrinsic);
+            let msg = format!(
+                "{intrinsic} is not yet supported.\n\
+                 See https://github.com/rust-lang/rustc_codegen_cranelift/issues/171\n\
+                 Please open an issue at https://github.com/rust-lang/rustc_codegen_cranelift/issues"
+            );
+            crate::base::codegen_panic_nounwind(fx, &msg, span);
             return;
         }
     }
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs
index 387c87d123a..3cd7ebb88f4 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs
@@ -7,7 +7,7 @@ use crate::inline_asm::{CInlineAsmOperand, codegen_inline_asm_inner};
 use crate::intrinsics::*;
 use crate::prelude::*;
 
-pub(crate) fn codegen_aarch64_llvm_intrinsic_call<'tcx>(
+pub(super) fn codegen_aarch64_llvm_intrinsic_call<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     intrinsic: &str,
     args: &[Spanned<mir::Operand<'tcx>>],
@@ -264,14 +264,6 @@ pub(crate) fn codegen_aarch64_llvm_intrinsic_call<'tcx>(
             simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().fadd(a, b));
         }
 
-        _ if intrinsic.starts_with("llvm.aarch64.neon.frintn.v") => {
-            intrinsic_args!(fx, args => (v); intrinsic);
-
-            simd_for_each_lane(fx, v, ret, &|fx, _lane_ty, _res_lane_ty, lane| {
-                fx.bcx.ins().nearest(lane)
-            });
-        }
-
         _ if intrinsic.starts_with("llvm.aarch64.neon.smaxv.i") => {
             intrinsic_args!(fx, args => (v); intrinsic);
 
@@ -507,7 +499,12 @@ pub(crate) fn codegen_aarch64_llvm_intrinsic_call<'tcx>(
                 "unsupported AArch64 llvm intrinsic {}; replacing with trap",
                 intrinsic
             ));
-            crate::trap::trap_unimplemented(fx, intrinsic);
+            let msg = format!(
+                "{intrinsic} is not yet supported.\n\
+                 See https://github.com/rust-lang/rustc_codegen_cranelift/issues/171\n\
+                 Please open an issue at https://github.com/rust-lang/rustc_codegen_cranelift/issues"
+            );
+            crate::base::codegen_panic_nounwind(fx, &msg, fx.mir.span);
             return;
         }
     }
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
index c02d31844e0..615f6c47d90 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
@@ -7,7 +7,7 @@ use crate::inline_asm::{CInlineAsmOperand, codegen_inline_asm_inner};
 use crate::intrinsics::*;
 use crate::prelude::*;
 
-pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
+pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     intrinsic: &str,
     args: &[Spanned<mir::Operand<'tcx>>],
@@ -147,10 +147,10 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
                 let offset = fx.bcx.ins().imul(index_lane, scale);
                 let lane_ptr = fx.bcx.ins().iadd(ptr, offset);
                 let res = fx.bcx.ins().load(lane_clif_ty, MemFlags::trusted(), lane_ptr, 0);
-                fx.bcx.ins().jump(next, &[res]);
+                fx.bcx.ins().jump(next, &[res.into()]);
 
                 fx.bcx.switch_to_block(if_disabled);
-                fx.bcx.ins().jump(next, &[src_lane]);
+                fx.bcx.ins().jump(next, &[src_lane.into()]);
 
                 fx.bcx.seal_block(next);
                 fx.bcx.switch_to_block(next);
@@ -1316,7 +1316,12 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             fx.tcx
                 .dcx()
                 .warn(format!("unsupported x86 llvm intrinsic {}; replacing with trap", intrinsic));
-            crate::trap::trap_unimplemented(fx, intrinsic);
+            let msg = format!(
+                "{intrinsic} is not yet supported.\n\
+                 See https://github.com/rust-lang/rustc_codegen_cranelift/issues/171\n\
+                 Please open an issue at https://github.com/rust-lang/rustc_codegen_cranelift/issues"
+            );
+            crate::base::codegen_panic_nounwind(fx, &msg, span);
             return;
         }
     }
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
index 9018d78b00a..1d1cf884e48 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
@@ -27,6 +27,7 @@ use rustc_span::{Symbol, sym};
 
 pub(crate) use self::llvm::codegen_llvm_intrinsic_call;
 use crate::cast::clif_intcast;
+use crate::codegen_f16_f128;
 use crate::prelude::*;
 
 fn bug_on_incorrect_arg_count(intrinsic: impl std::fmt::Display) -> ! {
@@ -248,8 +249,10 @@ fn bool_to_zero_or_max_uint<'tcx>(
     let ty = fx.clif_type(ty).unwrap();
 
     let int_ty = match ty {
+        types::F16 => types::I16,
         types::F32 => types::I32,
         types::F64 => types::I64,
+        types::F128 => types::I128,
         ty => ty,
     };
 
@@ -308,45 +311,83 @@ fn codegen_float_intrinsic_call<'tcx>(
     ret: CPlace<'tcx>,
 ) -> bool {
     let (name, arg_count, ty, clif_ty) = match intrinsic {
+        sym::expf16 => ("expf16", 1, fx.tcx.types.f16, types::F16),
         sym::expf32 => ("expf", 1, fx.tcx.types.f32, types::F32),
         sym::expf64 => ("exp", 1, fx.tcx.types.f64, types::F64),
+        sym::expf128 => ("expf128", 1, fx.tcx.types.f128, types::F128),
+        sym::exp2f16 => ("exp2f16", 1, fx.tcx.types.f16, types::F16),
         sym::exp2f32 => ("exp2f", 1, fx.tcx.types.f32, types::F32),
         sym::exp2f64 => ("exp2", 1, fx.tcx.types.f64, types::F64),
+        sym::exp2f128 => ("exp2f128", 1, fx.tcx.types.f128, types::F128),
+        sym::sqrtf16 => ("sqrtf16", 1, fx.tcx.types.f16, types::F16),
         sym::sqrtf32 => ("sqrtf", 1, fx.tcx.types.f32, types::F32),
         sym::sqrtf64 => ("sqrt", 1, fx.tcx.types.f64, types::F64),
+        sym::sqrtf128 => ("sqrtf128", 1, fx.tcx.types.f128, types::F128),
+        sym::powif16 => ("__powisf2", 2, fx.tcx.types.f16, types::F16), // compiler-builtins
         sym::powif32 => ("__powisf2", 2, fx.tcx.types.f32, types::F32), // compiler-builtins
         sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64, types::F64), // compiler-builtins
+        sym::powif128 => ("__powitf2", 2, fx.tcx.types.f128, types::F128), // compiler-builtins
+        sym::powf16 => ("powf16", 2, fx.tcx.types.f16, types::F16),
         sym::powf32 => ("powf", 2, fx.tcx.types.f32, types::F32),
         sym::powf64 => ("pow", 2, fx.tcx.types.f64, types::F64),
+        sym::powf128 => ("powf128", 2, fx.tcx.types.f128, types::F128),
+        sym::logf16 => ("logf16", 1, fx.tcx.types.f16, types::F16),
         sym::logf32 => ("logf", 1, fx.tcx.types.f32, types::F32),
         sym::logf64 => ("log", 1, fx.tcx.types.f64, types::F64),
+        sym::logf128 => ("logf128", 1, fx.tcx.types.f128, types::F128),
+        sym::log2f16 => ("log2f16", 1, fx.tcx.types.f16, types::F16),
         sym::log2f32 => ("log2f", 1, fx.tcx.types.f32, types::F32),
         sym::log2f64 => ("log2", 1, fx.tcx.types.f64, types::F64),
+        sym::log2f128 => ("log2f128", 1, fx.tcx.types.f128, types::F128),
+        sym::log10f16 => ("log10f16", 1, fx.tcx.types.f16, types::F16),
         sym::log10f32 => ("log10f", 1, fx.tcx.types.f32, types::F32),
         sym::log10f64 => ("log10", 1, fx.tcx.types.f64, types::F64),
+        sym::log10f128 => ("log10f128", 1, fx.tcx.types.f128, types::F128),
+        sym::fabsf16 => ("fabsf16", 1, fx.tcx.types.f16, types::F16),
         sym::fabsf32 => ("fabsf", 1, fx.tcx.types.f32, types::F32),
         sym::fabsf64 => ("fabs", 1, fx.tcx.types.f64, types::F64),
+        sym::fabsf128 => ("fabsf128", 1, fx.tcx.types.f128, types::F128),
+        sym::fmaf16 => ("fmaf16", 3, fx.tcx.types.f16, types::F16),
         sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32, types::F32),
         sym::fmaf64 => ("fma", 3, fx.tcx.types.f64, types::F64),
+        sym::fmaf128 => ("fmaf128", 3, fx.tcx.types.f128, types::F128),
         // FIXME: calling `fma` from libc without FMA target feature uses expensive sofware emulation
+        sym::fmuladdf16 => ("fmaf16", 3, fx.tcx.types.f16, types::F16), // TODO: use cranelift intrinsic analogous to llvm.fmuladd.f16
         sym::fmuladdf32 => ("fmaf", 3, fx.tcx.types.f32, types::F32), // TODO: use cranelift intrinsic analogous to llvm.fmuladd.f32
         sym::fmuladdf64 => ("fma", 3, fx.tcx.types.f64, types::F64), // TODO: use cranelift intrinsic analogous to llvm.fmuladd.f64
+        sym::fmuladdf128 => ("fmaf128", 3, fx.tcx.types.f128, types::F128), // TODO: use cranelift intrinsic analogous to llvm.fmuladd.f128
+        sym::copysignf16 => ("copysignf16", 2, fx.tcx.types.f16, types::F16),
         sym::copysignf32 => ("copysignf", 2, fx.tcx.types.f32, types::F32),
         sym::copysignf64 => ("copysign", 2, fx.tcx.types.f64, types::F64),
+        sym::copysignf128 => ("copysignf128", 2, fx.tcx.types.f128, types::F128),
+        sym::floorf16 => ("floorf16", 1, fx.tcx.types.f16, types::F16),
         sym::floorf32 => ("floorf", 1, fx.tcx.types.f32, types::F32),
         sym::floorf64 => ("floor", 1, fx.tcx.types.f64, types::F64),
+        sym::floorf128 => ("floorf128", 1, fx.tcx.types.f128, types::F128),
+        sym::ceilf16 => ("ceilf16", 1, fx.tcx.types.f16, types::F16),
         sym::ceilf32 => ("ceilf", 1, fx.tcx.types.f32, types::F32),
         sym::ceilf64 => ("ceil", 1, fx.tcx.types.f64, types::F64),
+        sym::ceilf128 => ("ceilf128", 1, fx.tcx.types.f128, types::F128),
+        sym::truncf16 => ("truncf16", 1, fx.tcx.types.f16, types::F16),
         sym::truncf32 => ("truncf", 1, fx.tcx.types.f32, types::F32),
         sym::truncf64 => ("trunc", 1, fx.tcx.types.f64, types::F64),
+        sym::truncf128 => ("truncf128", 1, fx.tcx.types.f128, types::F128),
+        sym::round_ties_even_f16 => ("rintf16", 1, fx.tcx.types.f16, types::F16),
         sym::round_ties_even_f32 => ("rintf", 1, fx.tcx.types.f32, types::F32),
         sym::round_ties_even_f64 => ("rint", 1, fx.tcx.types.f64, types::F64),
+        sym::round_ties_even_f128 => ("rintf128", 1, fx.tcx.types.f128, types::F128),
+        sym::roundf16 => ("roundf16", 1, fx.tcx.types.f16, types::F16),
         sym::roundf32 => ("roundf", 1, fx.tcx.types.f32, types::F32),
         sym::roundf64 => ("round", 1, fx.tcx.types.f64, types::F64),
+        sym::roundf128 => ("roundf128", 1, fx.tcx.types.f128, types::F128),
+        sym::sinf16 => ("sinf16", 1, fx.tcx.types.f16, types::F16),
         sym::sinf32 => ("sinf", 1, fx.tcx.types.f32, types::F32),
         sym::sinf64 => ("sin", 1, fx.tcx.types.f64, types::F64),
+        sym::sinf128 => ("sinf128", 1, fx.tcx.types.f128, types::F128),
+        sym::cosf16 => ("cosf16", 1, fx.tcx.types.f16, types::F16),
         sym::cosf32 => ("cosf", 1, fx.tcx.types.f32, types::F32),
         sym::cosf64 => ("cos", 1, fx.tcx.types.f64, types::F64),
+        sym::cosf128 => ("cosf128", 1, fx.tcx.types.f128, types::F128),
         _ => return false,
     };
 
@@ -379,13 +420,26 @@ fn codegen_float_intrinsic_call<'tcx>(
     };
 
     let layout = fx.layout_of(ty);
+    // FIXME(bytecodealliance/wasmtime#8312): Use native Cranelift operations
+    // for `f16` and `f128` once the lowerings have been implemented in Cranelift.
     let res = match intrinsic {
+        sym::fmaf16 | sym::fmuladdf16 => {
+            CValue::by_val(codegen_f16_f128::fma_f16(fx, args[0], args[1], args[2]), layout)
+        }
         sym::fmaf32 | sym::fmaf64 | sym::fmuladdf32 | sym::fmuladdf64 => {
             CValue::by_val(fx.bcx.ins().fma(args[0], args[1], args[2]), layout)
         }
+        sym::copysignf16 => {
+            CValue::by_val(codegen_f16_f128::copysign_f16(fx, args[0], args[1]), layout)
+        }
+        sym::copysignf128 => {
+            CValue::by_val(codegen_f16_f128::copysign_f128(fx, args[0], args[1]), layout)
+        }
         sym::copysignf32 | sym::copysignf64 => {
             CValue::by_val(fx.bcx.ins().fcopysign(args[0], args[1]), layout)
         }
+        sym::fabsf16 => CValue::by_val(codegen_f16_f128::abs_f16(fx, args[0]), layout),
+        sym::fabsf128 => CValue::by_val(codegen_f16_f128::abs_f128(fx, args[0]), layout),
         sym::fabsf32
         | sym::fabsf64
         | sym::floorf32
@@ -415,11 +469,36 @@ fn codegen_float_intrinsic_call<'tcx>(
 
         // These intrinsics aren't supported natively by Cranelift.
         // Lower them to a libcall.
-        sym::powif32 | sym::powif64 => {
-            let input_tys: Vec<_> = vec![AbiParam::new(clif_ty), AbiParam::new(types::I32)];
+        sym::powif16 | sym::powif32 | sym::powif64 | sym::powif128 => {
+            let temp;
+            let (clif_ty, args) = if intrinsic == sym::powif16 {
+                temp = [codegen_f16_f128::f16_to_f32(fx, args[0]), args[1]];
+                (types::F32, temp.as_slice())
+            } else {
+                (clif_ty, args)
+            };
+            let input_tys: Vec<_> =
+                vec![AbiParam::new(clif_ty), lib_call_arg_param(fx.tcx, types::I32, true)];
             let ret_val = fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], &args)[0];
+            let ret_val = if intrinsic == sym::powif16 {
+                codegen_f16_f128::f32_to_f16(fx, ret_val)
+            } else {
+                ret_val
+            };
             CValue::by_val(ret_val, fx.layout_of(ty))
         }
+        sym::powf16 => {
+            // FIXME(f16_f128): Rust `compiler-builtins` doesn't export `powf16` yet.
+            let x = codegen_f16_f128::f16_to_f32(fx, args[0]);
+            let y = codegen_f16_f128::f16_to_f32(fx, args[1]);
+            let ret_val = fx.lib_call(
+                "powf",
+                vec![AbiParam::new(types::F32), AbiParam::new(types::F32)],
+                vec![AbiParam::new(types::F32)],
+                &[x, y],
+            )[0];
+            CValue::by_val(codegen_f16_f128::f32_to_f16(fx, ret_val), fx.layout_of(ty))
+        }
         _ => {
             let input_tys: Vec<_> = args.iter().map(|_| AbiParam::new(clif_ty)).collect();
             let ret_val = fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], &args)[0];
@@ -706,7 +785,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
                             }
                         })
                     });
-                    crate::base::codegen_panic_nounwind(fx, &msg_str, Some(source_info.span));
+                    crate::base::codegen_panic_nounwind(fx, &msg_str, source_info.span);
                     return Ok(());
                 }
             }
@@ -733,11 +812,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             dest.write_cvalue(fx, val);
         }
 
-        sym::pref_align_of
-        | sym::needs_drop
-        | sym::type_id
-        | sym::type_name
-        | sym::variant_count => {
+        sym::needs_drop | sym::type_id | sym::type_name | sym::variant_count => {
             intrinsic_args!(fx, args => (); intrinsic);
 
             let const_val = fx
@@ -791,7 +866,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             // FIXME use a compiler fence once Cranelift supports it
             fx.bcx.ins().fence();
         }
-        _ if intrinsic.as_str().starts_with("atomic_load") => {
+        sym::atomic_load => {
             intrinsic_args!(fx, args => (ptr); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -801,7 +876,11 @@ fn codegen_regular_intrinsic_call<'tcx>(
                     // FIXME implement 128bit atomics
                     if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
                         // special case for compiler-builtins to avoid having to patch it
-                        crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
+                        crate::base::codegen_panic_nounwind(
+                            fx,
+                            "128bit atomics not yet supported",
+                            source_info.span,
+                        );
                         return Ok(());
                     } else {
                         fx.tcx
@@ -822,7 +901,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let val = CValue::by_val(val, fx.layout_of(ty));
             ret.write_cvalue(fx, val);
         }
-        _ if intrinsic.as_str().starts_with("atomic_store") => {
+        sym::atomic_store => {
             intrinsic_args!(fx, args => (ptr, val); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -832,7 +911,11 @@ fn codegen_regular_intrinsic_call<'tcx>(
                     // FIXME implement 128bit atomics
                     if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
                         // special case for compiler-builtins to avoid having to patch it
-                        crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
+                        crate::base::codegen_panic_nounwind(
+                            fx,
+                            "128bit atomics not yet supported",
+                            source_info.span,
+                        );
                         return Ok(());
                     } else {
                         fx.tcx
@@ -851,7 +934,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
 
             fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr);
         }
-        _ if intrinsic.as_str().starts_with("atomic_xchg") => {
+        sym::atomic_xchg => {
             intrinsic_args!(fx, args => (ptr, new); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -872,8 +955,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         }
-        _ if intrinsic.as_str().starts_with("atomic_cxchg") => {
-            // both atomic_cxchg_* and atomic_cxchgweak_*
+        sym::atomic_cxchg | sym::atomic_cxchgweak => {
             intrinsic_args!(fx, args => (ptr, test_old, new); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -896,7 +978,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, ret_val)
         }
 
-        _ if intrinsic.as_str().starts_with("atomic_xadd") => {
+        sym::atomic_xadd => {
             intrinsic_args!(fx, args => (ptr, amount); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -918,7 +1000,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         }
-        _ if intrinsic.as_str().starts_with("atomic_xsub") => {
+        sym::atomic_xsub => {
             intrinsic_args!(fx, args => (ptr, amount); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -940,7 +1022,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         }
-        _ if intrinsic.as_str().starts_with("atomic_and") => {
+        sym::atomic_and => {
             intrinsic_args!(fx, args => (ptr, src); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -961,7 +1043,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         }
-        _ if intrinsic.as_str().starts_with("atomic_or") => {
+        sym::atomic_or => {
             intrinsic_args!(fx, args => (ptr, src); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -982,7 +1064,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         }
-        _ if intrinsic.as_str().starts_with("atomic_xor") => {
+        sym::atomic_xor => {
             intrinsic_args!(fx, args => (ptr, src); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -1003,7 +1085,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         }
-        _ if intrinsic.as_str().starts_with("atomic_nand") => {
+        sym::atomic_nand => {
             intrinsic_args!(fx, args => (ptr, src); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -1024,7 +1106,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         }
-        _ if intrinsic.as_str().starts_with("atomic_max") => {
+        sym::atomic_max => {
             intrinsic_args!(fx, args => (ptr, src); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -1045,7 +1127,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         }
-        _ if intrinsic.as_str().starts_with("atomic_umax") => {
+        sym::atomic_umax => {
             intrinsic_args!(fx, args => (ptr, src); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -1066,7 +1148,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         }
-        _ if intrinsic.as_str().starts_with("atomic_min") => {
+        sym::atomic_min => {
             intrinsic_args!(fx, args => (ptr, src); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -1087,7 +1169,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let old = CValue::by_val(old, layout);
             ret.write_cvalue(fx, old);
         }
-        _ if intrinsic.as_str().starts_with("atomic_umin") => {
+        sym::atomic_umin => {
             intrinsic_args!(fx, args => (ptr, src); intrinsic);
             let ptr = ptr.load_scalar(fx);
 
@@ -1109,6 +1191,20 @@ fn codegen_regular_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, old);
         }
 
+        sym::minimumf16 => {
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+            let a = a.load_scalar(fx);
+            let b = b.load_scalar(fx);
+
+            // FIXME(bytecodealliance/wasmtime#8312): Use `fmin` directly once
+            // Cranelift backend lowerings are implemented.
+            let a = codegen_f16_f128::f16_to_f32(fx, a);
+            let b = codegen_f16_f128::f16_to_f32(fx, b);
+            let val = fx.bcx.ins().fmin(a, b);
+            let val = codegen_f16_f128::f32_to_f16(fx, val);
+            let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f16));
+            ret.write_cvalue(fx, val);
+        }
         sym::minimumf32 => {
             intrinsic_args!(fx, args => (a, b); intrinsic);
             let a = a.load_scalar(fx);
@@ -1127,6 +1223,31 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
             ret.write_cvalue(fx, val);
         }
+        sym::minimumf128 => {
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+            let a = a.load_scalar(fx);
+            let b = b.load_scalar(fx);
+
+            // FIXME(bytecodealliance/wasmtime#8312): Use `fmin` once  Cranelift
+            // backend lowerings are implemented.
+            let val = codegen_f16_f128::fmin_f128(fx, a, b);
+            let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f128));
+            ret.write_cvalue(fx, val);
+        }
+        sym::maximumf16 => {
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+            let a = a.load_scalar(fx);
+            let b = b.load_scalar(fx);
+
+            // FIXME(bytecodealliance/wasmtime#8312): Use `fmax` directly once
+            // Cranelift backend lowerings are implemented.
+            let a = codegen_f16_f128::f16_to_f32(fx, a);
+            let b = codegen_f16_f128::f16_to_f32(fx, b);
+            let val = fx.bcx.ins().fmax(a, b);
+            let val = codegen_f16_f128::f32_to_f16(fx, val);
+            let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f16));
+            ret.write_cvalue(fx, val);
+        }
         sym::maximumf32 => {
             intrinsic_args!(fx, args => (a, b); intrinsic);
             let a = a.load_scalar(fx);
@@ -1145,7 +1266,27 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
             ret.write_cvalue(fx, val);
         }
+        sym::maximumf128 => {
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+            let a = a.load_scalar(fx);
+            let b = b.load_scalar(fx);
+
+            // FIXME(bytecodealliance/wasmtime#8312): Use `fmax` once  Cranelift
+            // backend lowerings are implemented.
+            let val = codegen_f16_f128::fmax_f128(fx, a, b);
+            let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f128));
+            ret.write_cvalue(fx, val);
+        }
+
+        sym::minnumf16 => {
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+            let a = a.load_scalar(fx);
+            let b = b.load_scalar(fx);
 
+            let val = crate::num::codegen_float_min(fx, a, b);
+            let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f16));
+            ret.write_cvalue(fx, val);
+        }
         sym::minnumf32 => {
             intrinsic_args!(fx, args => (a, b); intrinsic);
             let a = a.load_scalar(fx);
@@ -1164,6 +1305,24 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
             ret.write_cvalue(fx, val);
         }
+        sym::minnumf128 => {
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+            let a = a.load_scalar(fx);
+            let b = b.load_scalar(fx);
+
+            let val = crate::num::codegen_float_min(fx, a, b);
+            let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f128));
+            ret.write_cvalue(fx, val);
+        }
+        sym::maxnumf16 => {
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+            let a = a.load_scalar(fx);
+            let b = b.load_scalar(fx);
+
+            let val = crate::num::codegen_float_max(fx, a, b);
+            let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f16));
+            ret.write_cvalue(fx, val);
+        }
         sym::maxnumf32 => {
             intrinsic_args!(fx, args => (a, b); intrinsic);
             let a = a.load_scalar(fx);
@@ -1182,6 +1341,15 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
             ret.write_cvalue(fx, val);
         }
+        sym::maxnumf128 => {
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+            let a = a.load_scalar(fx);
+            let b = b.load_scalar(fx);
+
+            let val = crate::num::codegen_float_max(fx, a, b);
+            let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f128));
+            ret.write_cvalue(fx, val);
+        }
 
         sym::catch_unwind => {
             intrinsic_args!(fx, args => (f, data, catch_fn); intrinsic);
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
index dd6d8dbb6f5..46a441488fa 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
@@ -283,6 +283,20 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             ret_lane.write_cvalue(fx, val);
         }
 
+        sym::simd_insert_dyn => {
+            intrinsic_args!(fx, args => (base, idx, val); intrinsic);
+
+            if !base.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, base.layout().ty);
+                return;
+            }
+
+            let idx = idx.load_scalar(fx);
+
+            ret.write_cvalue(fx, base);
+            ret.write_lane_dyn(fx, idx, val);
+        }
+
         sym::simd_extract => {
             let (v, idx) = match args {
                 [v, idx] => (v, idx),
@@ -318,6 +332,20 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, ret_lane);
         }
 
+        sym::simd_extract_dyn => {
+            intrinsic_args!(fx, args => (v, idx); intrinsic);
+
+            if !v.layout().ty.is_simd() {
+                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
+                return;
+            }
+
+            let idx = idx.load_scalar(fx);
+
+            let ret_lane = v.value_lane_dyn(fx, idx);
+            ret.write_cvalue(fx, ret_lane);
+        }
+
         sym::simd_neg
         | sym::simd_bswap
         | sym::simd_bitreverse
@@ -980,10 +1008,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
                 fx.bcx.switch_to_block(if_enabled);
                 let res = fx.bcx.ins().load(lane_clif_ty, MemFlags::trusted(), ptr_lane, 0);
-                fx.bcx.ins().jump(next, &[res]);
+                fx.bcx.ins().jump(next, &[res.into()]);
 
                 fx.bcx.switch_to_block(if_disabled);
-                fx.bcx.ins().jump(next, &[val_lane]);
+                fx.bcx.ins().jump(next, &[val_lane.into()]);
 
                 fx.bcx.seal_block(next);
                 fx.bcx.switch_to_block(next);
@@ -1029,10 +1057,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                     ptr_val,
                     Offset32::new(offset),
                 );
-                fx.bcx.ins().jump(next, &[res]);
+                fx.bcx.ins().jump(next, &[res.into()]);
 
                 fx.bcx.switch_to_block(if_disabled);
-                fx.bcx.ins().jump(next, &[val_lane]);
+                fx.bcx.ins().jump(next, &[val_lane.into()]);
 
                 fx.bcx.seal_block(next);
                 fx.bcx.switch_to_block(next);
diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs
index ab09a6f8b38..8ef623cde00 100644
--- a/compiler/rustc_codegen_cranelift/src/lib.rs
+++ b/compiler/rustc_codegen_cranelift/src/lib.rs
@@ -6,6 +6,9 @@
 #![cfg_attr(doc, feature(rustdoc_internals))]
 // Note: please avoid adding other feature gates where possible
 #![feature(rustc_private)]
+// Only used to define intrinsics in `compiler_builtins.rs`.
+#![feature(f16)]
+#![feature(f128)]
 // Note: please avoid adding other feature gates where possible
 #![warn(rust_2018_idioms)]
 #![warn(unreachable_pub)]
@@ -57,6 +60,7 @@ mod allocator;
 mod analyze;
 mod base;
 mod cast;
+mod codegen_f16_f128;
 mod codegen_i128;
 mod common;
 mod compiler_builtins;
@@ -76,7 +80,6 @@ mod optimize;
 mod pointer;
 mod pretty_clif;
 mod toolchain;
-mod trap;
 mod unsize;
 mod unwind_module;
 mod value_and_place;
@@ -198,14 +201,36 @@ impl CodegenBackend for CraneliftCodegenBackend {
         // FIXME do `unstable_target_features` properly
         let unstable_target_features = target_features.clone();
 
+        // FIXME(f16_f128): LLVM 20 (currently used by `rustc`) passes `f128` in XMM registers on
+        // Windows, whereas LLVM 21+ and Cranelift pass it indirectly. This means that `f128` won't
+        // work when linking against a LLVM-built sysroot.
+        let has_reliable_f128 = !sess.target.is_like_windows;
+        let has_reliable_f16 = match &*sess.target.arch {
+            // FIXME(f16_f128): LLVM 20 does not support `f16` on s390x, meaning the required
+            // builtins are not available in `compiler-builtins`.
+            "s390x" => false,
+            // FIXME(f16_f128): `rustc_codegen_llvm` currently disables support on Windows GNU
+            // targets due to GCC using a different ABI than LLVM. Therefore `f16` won't be
+            // available when using a LLVM-built sysroot.
+            "x86_64"
+                if sess.target.os == "windows"
+                    && sess.target.env == "gnu"
+                    && sess.target.abi != "llvm" =>
+            {
+                false
+            }
+            _ => true,
+        };
+
         TargetConfig {
             target_features,
             unstable_target_features,
-            // Cranelift does not yet support f16 or f128
-            has_reliable_f16: false,
-            has_reliable_f16_math: false,
-            has_reliable_f128: false,
-            has_reliable_f128_math: false,
+            // `rustc_codegen_cranelift` polyfills functionality not yet
+            // available in Cranelift.
+            has_reliable_f16,
+            has_reliable_f16_math: has_reliable_f16,
+            has_reliable_f128,
+            has_reliable_f128_math: has_reliable_f128,
         }
     }
 
@@ -290,6 +315,12 @@ fn build_isa(sess: &Session, jit: bool) -> Arc<dyn TargetIsa + 'static> {
 
     flags_builder.set("enable_llvm_abi_extensions", "true").unwrap();
 
+    if let Some(align) = sess.opts.unstable_opts.min_function_alignment {
+        flags_builder
+            .set("log2_min_function_alignment", &align.bytes().ilog2().to_string())
+            .unwrap();
+    }
+
     use rustc_session::config::OptLevel;
     match sess.opts.optimize {
         OptLevel::No => {
diff --git a/compiler/rustc_codegen_cranelift/src/main_shim.rs b/compiler/rustc_codegen_cranelift/src/main_shim.rs
index 6eef97c14dd..bf756860b64 100644
--- a/compiler/rustc_codegen_cranelift/src/main_shim.rs
+++ b/compiler/rustc_codegen_cranelift/src/main_shim.rs
@@ -101,7 +101,7 @@ pub(crate) fn maybe_create_entry_wrapper(
                 let call_inst = bcx.ins().call(main_func_ref, &[]);
                 let call_results = bcx.func.dfg.inst_results(call_inst).to_owned();
 
-                let termination_trait = tcx.require_lang_item(LangItem::Termination, None);
+                let termination_trait = tcx.require_lang_item(LangItem::Termination, DUMMY_SP);
                 let report = tcx
                     .associated_items(termination_trait)
                     .find_by_ident_and_kind(
@@ -136,7 +136,7 @@ pub(crate) fn maybe_create_entry_wrapper(
                 }
             } else {
                 // Regular main fn invoked via start lang item.
-                let start_def_id = tcx.require_lang_item(LangItem::Start, None);
+                let start_def_id = tcx.require_lang_item(LangItem::Start, DUMMY_SP);
                 let start_instance = Instance::expect_resolve(
                     tcx,
                     ty::TypingEnv::fully_monomorphized(),
diff --git a/compiler/rustc_codegen_cranelift/src/num.rs b/compiler/rustc_codegen_cranelift/src/num.rs
index 2a4d1e3ae57..95d44dfb6d9 100644
--- a/compiler/rustc_codegen_cranelift/src/num.rs
+++ b/compiler/rustc_codegen_cranelift/src/num.rs
@@ -1,8 +1,9 @@
 //! Various operations on integer and floating-point numbers
 
+use crate::codegen_f16_f128;
 use crate::prelude::*;
 
-pub(crate) fn bin_op_to_intcc(bin_op: BinOp, signed: bool) -> IntCC {
+fn bin_op_to_intcc(bin_op: BinOp, signed: bool) -> IntCC {
     use BinOp::*;
     use IntCC::*;
     match bin_op {
@@ -53,7 +54,7 @@ fn codegen_three_way_compare<'tcx>(
     let gt = fx.bcx.ins().icmp(gt_cc, lhs, rhs);
     let lt = fx.bcx.ins().icmp(lt_cc, lhs, rhs);
     let val = fx.bcx.ins().isub(gt, lt);
-    CValue::by_val(val, fx.layout_of(fx.tcx.ty_ordering_enum(Some(fx.mir.span))))
+    CValue::by_val(val, fx.layout_of(fx.tcx.ty_ordering_enum(fx.mir.span)))
 }
 
 fn codegen_compare_bin_op<'tcx>(
@@ -109,7 +110,7 @@ pub(crate) fn codegen_binop<'tcx>(
     }
 }
 
-pub(crate) fn codegen_bool_binop<'tcx>(
+fn codegen_bool_binop<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     bin_op: BinOp,
     in_lhs: CValue<'tcx>,
@@ -350,25 +351,60 @@ pub(crate) fn codegen_float_binop<'tcx>(
     let lhs = in_lhs.load_scalar(fx);
     let rhs = in_rhs.load_scalar(fx);
 
+    // FIXME(bytecodealliance/wasmtime#8312): Remove once backend lowerings have
+    // been added to Cranelift.
+    let (lhs, rhs) = if *in_lhs.layout().ty.kind() == ty::Float(FloatTy::F16) {
+        (codegen_f16_f128::f16_to_f32(fx, lhs), codegen_f16_f128::f16_to_f32(fx, rhs))
+    } else {
+        (lhs, rhs)
+    };
     let b = fx.bcx.ins();
     let res = match bin_op {
+        // FIXME(bytecodealliance/wasmtime#8312): Remove once backend lowerings
+        // have been added to Cranelift.
+        BinOp::Add | BinOp::Sub | BinOp::Mul | BinOp::Div
+            if *in_lhs.layout().ty.kind() == ty::Float(FloatTy::F128) =>
+        {
+            codegen_f16_f128::codegen_f128_binop(fx, bin_op, lhs, rhs)
+        }
         BinOp::Add => b.fadd(lhs, rhs),
         BinOp::Sub => b.fsub(lhs, rhs),
         BinOp::Mul => b.fmul(lhs, rhs),
         BinOp::Div => b.fdiv(lhs, rhs),
         BinOp::Rem => {
-            let (name, ty) = match in_lhs.layout().ty.kind() {
-                ty::Float(FloatTy::F32) => ("fmodf", types::F32),
-                ty::Float(FloatTy::F64) => ("fmod", types::F64),
+            let (name, ty, lhs, rhs) = match in_lhs.layout().ty.kind() {
+                ty::Float(FloatTy::F16) => (
+                    "fmodf",
+                    types::F32,
+                    // FIXME(bytecodealliance/wasmtime#8312): Already converted
+                    // by the FIXME above.
+                    // fx.bcx.ins().fpromote(types::F32, lhs),
+                    // fx.bcx.ins().fpromote(types::F32, rhs),
+                    lhs,
+                    rhs,
+                ),
+                ty::Float(FloatTy::F32) => ("fmodf", types::F32, lhs, rhs),
+                ty::Float(FloatTy::F64) => ("fmod", types::F64, lhs, rhs),
+                ty::Float(FloatTy::F128) => ("fmodf128", types::F128, lhs, rhs),
                 _ => bug!(),
             };
 
-            fx.lib_call(
+            let ret_val = fx.lib_call(
                 name,
                 vec![AbiParam::new(ty), AbiParam::new(ty)],
                 vec![AbiParam::new(ty)],
                 &[lhs, rhs],
-            )[0]
+            )[0];
+
+            let ret_val = if *in_lhs.layout().ty.kind() == ty::Float(FloatTy::F16) {
+                // FIXME(bytecodealliance/wasmtime#8312): Use native Cranelift
+                // operation once Cranelift backend lowerings have been
+                // implemented.
+                codegen_f16_f128::f32_to_f16(fx, ret_val)
+            } else {
+                ret_val
+            };
+            return CValue::by_val(ret_val, in_lhs.layout());
         }
         BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => {
             let fltcc = match bin_op {
@@ -380,16 +416,26 @@ pub(crate) fn codegen_float_binop<'tcx>(
                 BinOp::Gt => FloatCC::GreaterThan,
                 _ => unreachable!(),
             };
-            let val = fx.bcx.ins().fcmp(fltcc, lhs, rhs);
+            // FIXME(bytecodealliance/wasmtime#8312): Replace with Cranelift
+            // `fcmp` once `f16`/`f128` backend lowerings have been added to
+            // Cranelift.
+            let val = codegen_f16_f128::fcmp(fx, fltcc, lhs, rhs);
             return CValue::by_val(val, fx.layout_of(fx.tcx.types.bool));
         }
         _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs),
     };
 
+    // FIXME(bytecodealliance/wasmtime#8312): Remove once backend lowerings have
+    // been added to Cranelift.
+    let res = if *in_lhs.layout().ty.kind() == ty::Float(FloatTy::F16) {
+        codegen_f16_f128::f32_to_f16(fx, res)
+    } else {
+        res
+    };
     CValue::by_val(res, in_lhs.layout())
 }
 
-pub(crate) fn codegen_ptr_binop<'tcx>(
+fn codegen_ptr_binop<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     bin_op: BinOp,
     in_lhs: CValue<'tcx>,
@@ -457,15 +503,19 @@ pub(crate) fn codegen_ptr_binop<'tcx>(
 // and `a.is_nan() ? b : (a <= b ? b : a)` for `maxnumf*`. NaN checks are done by comparing
 // a float against itself. Only in case of NaN is it not equal to itself.
 pub(crate) fn codegen_float_min(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value {
-    let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a);
-    let a_ge_b = fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, a, b);
+    // FIXME(bytecodealliance/wasmtime#8312): Replace with Cranelift `fcmp` once
+    // `f16`/`f128` backend lowerings have been added to Cranelift.
+    let a_is_nan = codegen_f16_f128::fcmp(fx, FloatCC::NotEqual, a, a);
+    let a_ge_b = codegen_f16_f128::fcmp(fx, FloatCC::GreaterThanOrEqual, a, b);
     let temp = fx.bcx.ins().select(a_ge_b, b, a);
     fx.bcx.ins().select(a_is_nan, b, temp)
 }
 
 pub(crate) fn codegen_float_max(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value {
-    let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a);
-    let a_le_b = fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, a, b);
+    // FIXME(bytecodealliance/wasmtime#8312): Replace with Cranelift `fcmp` once
+    // `f16`/`f128` backend lowerings have been added to Cranelift.
+    let a_is_nan = codegen_f16_f128::fcmp(fx, FloatCC::NotEqual, a, a);
+    let a_le_b = codegen_f16_f128::fcmp(fx, FloatCC::LessThanOrEqual, a, b);
     let temp = fx.bcx.ins().select(a_le_b, b, a);
     fx.bcx.ins().select(a_is_nan, b, temp)
 }
diff --git a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
index cd254b04ed9..9400ae9fcff 100644
--- a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
+++ b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
@@ -8,40 +8,41 @@
 //! target x86_64
 //!
 //! function u0:22(i64) -> i8, i8 system_v {
-//! ; symbol _ZN97_$LT$example..IsNotEmpty$u20$as$u20$mini_core..FnOnce$LT$$LP$$RF$$RF$$u5b$u16$u5d$$C$$RP$$GT$$GT$9call_once17hd517c453d67c0915E
-//! ; instance Instance { def: Item(WithOptConstParam { did: DefId(0:42 ~ example[4e51]::{impl#0}::call_once), const_param_did: None }), args: [ReErased, ReErased] }
-//! ; abi FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: IsNotEmpty, layout: Layout { size: Size(0 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, abi: Aggregate { sized: true }, fields: Arbitrary { offsets: [], memory_index: [] }, largest_niche: None, variants: Single { index: 0 } } }, mode: Ignore }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, abi: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), variants: Single { index: 0 } } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, abi: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], memory_index: [0, 1] }, largest_niche: None, variants: Single { index: 0 } } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false }
+//! ; symbol _ZN97_$LT$example..IsNotEmpty$u20$as$u20$mini_core..FnOnce$LT$$LP$$RF$$RF$$u5b$u16$u5d$$C$$RP$$GT$$GT$9call_once17hd361e9f5c3d1c4deE
+//! ; instance Instance { def: Item(DefId(0:42 ~ example[3895]::{impl#0}::call_once)), args: ['{erased}, '{erased}] }
+//! ; abi FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: IsNotEmpty, layout: Layout { size: Size(0 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: Memory { sized: true }, fields: Arbitrary { offsets: [], memory_index: [] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 12266848898570219025 } }, mode: Ignore }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], memory_index: [0, 1] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 71776127651151873 } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false }
 //!
 //! ; kind  loc.idx   param    pass mode                            ty
-//! ; ssa   _0    (u8, u8)                          2b 1, 8              var=(0, 1)
+//! ; ssa   _0    (u8, u8)                          2b 1                var=(0, 1)
 //! ; ret   _0      -          Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) (u8, u8)
 //! ; arg   _1      -          Ignore                               IsNotEmpty
-//! ; arg   _2.0    = v0       Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) &&[u16]
+//! ; arg   _2.0    = v0       Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) &'{erased} &'{erased} [u16]
 //!
-//! ; kind  local ty                              size align (abi,pref)
-//! ; zst   _1    IsNotEmpty                        0b 1, 8              align=8,offset=
-//! ; stack _2    (&&[u16],)                        8b 8, 8              storage=ss0
-//! ; ssa   _3    &mut IsNotEmpty                   8b 8, 8              var=2
+//! ; kind  local ty                              size align (abi)
+//! ; zst   _1    IsNotEmpty                        0b 1                align=1,offset=
+//! ; stack _2    (&'{erased} &'{erased} [u16],)    8b 8                storage=ss0
+//! ; ssa   _3    &'{erased} mut IsNotEmpty         8b 8                var=2
 //!
-//!     ss0 = explicit_slot 16
+//!     ss0 = explicit_slot 16, align = 16
 //!     sig0 = (i64, i64) -> i8, i8 system_v
-//!     fn0 = colocated u0:23 sig0 ; Instance { def: Item(WithOptConstParam { did: DefId(0:46 ~ example[4e51]::{impl#1}::call_mut), const_param_did: None }), args: [ReErased, ReErased] }
+//!     fn0 = colocated u0:23 sig0 ; Instance { def: Item(DefId(0:46 ~ example[3895]::{impl#1}::call_mut)), args: ['{erased}, '{erased}] }
 //!
 //! block0(v0: i64):
 //!     nop
-//! ; write_cvalue: Addr(Pointer { base: Stack(ss0), offset: Offset32(0) }, None): &&[u16] <- ByVal(v0): &&[u16]
+//! ; write_cvalue: Addr(Pointer { base: Stack(ss0), offset: Offset32(0) }, None): &'{erased} &'{erased} [u16] <- ByVal(v0): &'{erased} &'{erased} [u16]
 //!     stack_store v0, ss0
 //!     jump block1
 //!
 //! block1:
 //!     nop
 //! ; _3 = &mut _1
-//!     v1 = iconst.i64 8
-//! ; write_cvalue: Var(_3, var2): &mut IsNotEmpty <- ByVal(v1): &mut IsNotEmpty
+//!     v1 = iconst.i64 1
+//! ; write_cvalue: Var(_3, var2): &'{erased} mut IsNotEmpty <- ByVal(v1): &'{erased} mut IsNotEmpty
 //! ;
-//! ; _0 = <IsNotEmpty as mini_core::FnMut<(&&[u16],)>>::call_mut(move _3, _2)
+//! ; _0 = <IsNotEmpty as mini_core::FnMut<(&&[u16],)>>::call_mut(move _3, copy _2)
 //!     v2 = stack_load.i64 ss0
-//!     v3, v4 = call fn0(v1, v2)  ; v1 = 8
+//! ; abi: FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: &mut IsNotEmpty, layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(1 bytes)) }) }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], memory_index: [0, 1] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 71776127651151873 } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false }
+//!     v3, v4 = call fn0(v1, v2)  ; v1 = 1
 //!     v5 -> v3
 //!     v6 -> v4
 //! ; write_cvalue: VarPair(_0, var0, var1): (u8, u8) <- ByValPair(v3, v4): (u8, u8)
@@ -73,6 +74,7 @@ pub(crate) struct CommentWriter {
     enabled: bool,
     global_comments: Vec<String>,
     entity_comments: FxHashMap<AnyEntity, String>,
+    inst_post_comments: FxHashMap<Inst, String>,
 }
 
 impl CommentWriter {
@@ -95,7 +97,12 @@ impl CommentWriter {
             vec![]
         };
 
-        CommentWriter { enabled, global_comments, entity_comments: FxHashMap::default() }
+        CommentWriter {
+            enabled,
+            global_comments,
+            entity_comments: FxHashMap::default(),
+            inst_post_comments: FxHashMap::default(),
+        }
     }
 }
 
@@ -127,6 +134,25 @@ impl CommentWriter {
             }
         }
     }
+
+    pub(crate) fn add_post_comment<S: Into<String> + AsRef<str>>(
+        &mut self,
+        entity: Inst,
+        comment: S,
+    ) {
+        debug_assert!(self.enabled);
+
+        use std::collections::hash_map::Entry;
+        match self.inst_post_comments.entry(entity) {
+            Entry::Occupied(mut occ) => {
+                occ.get_mut().push('\n');
+                occ.get_mut().push_str(comment.as_ref());
+            }
+            Entry::Vacant(vac) => {
+                vac.insert(comment.into());
+            }
+        }
+    }
 }
 
 impl FuncWriter for &'_ CommentWriter {
@@ -188,10 +214,13 @@ impl FuncWriter for &'_ CommentWriter {
         inst: Inst,
         indent: usize,
     ) -> fmt::Result {
-        PlainWriter.write_instruction(w, func, aliases, inst, indent)?;
         if let Some(comment) = self.entity_comments.get(&inst.into()) {
             writeln!(w, "; {}", comment.replace('\n', "\n; "))?;
         }
+        PlainWriter.write_instruction(w, func, aliases, inst, indent)?;
+        if let Some(comment) = self.inst_post_comments.get(&inst) {
+            writeln!(w, "; {}", comment.replace('\n', "\n; "))?;
+        }
         Ok(())
     }
 }
@@ -208,6 +237,14 @@ impl FunctionCx<'_, '_, '_> {
     ) {
         self.clif_comments.add_comment(entity, comment);
     }
+
+    pub(crate) fn add_post_comment<S: Into<String> + AsRef<str>>(
+        &mut self,
+        entity: Inst,
+        comment: S,
+    ) {
+        self.clif_comments.add_post_comment(entity, comment);
+    }
 }
 
 pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool {
diff --git a/compiler/rustc_codegen_cranelift/src/trap.rs b/compiler/rustc_codegen_cranelift/src/trap.rs
deleted file mode 100644
index ac3f58ee1ee..00000000000
--- a/compiler/rustc_codegen_cranelift/src/trap.rs
+++ /dev/null
@@ -1,38 +0,0 @@
-//! Helpers used to print a message and abort in case of certain panics and some detected UB.
-
-use crate::prelude::*;
-
-fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) {
-    let puts = fx
-        .module
-        .declare_function(
-            "puts",
-            Linkage::Import,
-            &Signature {
-                call_conv: fx.target_config.default_call_conv,
-                params: vec![AbiParam::new(fx.pointer_type)],
-                returns: vec![AbiParam::new(types::I32)],
-            },
-        )
-        .unwrap();
-    let puts = fx.module.declare_func_in_func(puts, &mut fx.bcx.func);
-    if fx.clif_comments.enabled() {
-        fx.add_comment(puts, "puts");
-    }
-
-    let real_msg = format!("trap at {:?} ({}): {}\0", fx.instance, fx.symbol_name, msg);
-    let msg_ptr = fx.anonymous_str(&real_msg);
-    fx.bcx.ins().call(puts, &[msg_ptr]);
-}
-
-/// Use this when something is unimplemented, but `libcore` or `libstd` requires it to codegen.
-///
-/// Trap code: user65535
-pub(crate) fn trap_unimplemented(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) {
-    codegen_print(fx, msg.as_ref());
-
-    let one = fx.bcx.ins().iconst(types::I32, 1);
-    fx.lib_call("exit", vec![AbiParam::new(types::I32)], vec![], &[one]);
-
-    fx.bcx.ins().trap(TrapCode::user(3).unwrap());
-}
diff --git a/compiler/rustc_codegen_cranelift/src/unsize.rs b/compiler/rustc_codegen_cranelift/src/unsize.rs
index f8bbb214920..662546e4999 100644
--- a/compiler/rustc_codegen_cranelift/src/unsize.rs
+++ b/compiler/rustc_codegen_cranelift/src/unsize.rs
@@ -240,7 +240,7 @@ pub(crate) fn size_and_align_of<'tcx>(
                 })
             });
 
-            codegen_panic_nounwind(fx, &msg_str, None);
+            codegen_panic_nounwind(fx, &msg_str, fx.mir.span);
 
             fx.bcx.switch_to_block(next_block);
 
diff --git a/compiler/rustc_codegen_cranelift/src/unwind_module.rs b/compiler/rustc_codegen_cranelift/src/unwind_module.rs
index b950aaa29ce..b4eb939cf25 100644
--- a/compiler/rustc_codegen_cranelift/src/unwind_module.rs
+++ b/compiler/rustc_codegen_cranelift/src/unwind_module.rs
@@ -1,10 +1,10 @@
+use cranelift_codegen::Context;
 use cranelift_codegen::control::ControlPlane;
-use cranelift_codegen::ir::{Function, Signature};
+use cranelift_codegen::ir::Signature;
 use cranelift_codegen::isa::{TargetFrontendConfig, TargetIsa};
-use cranelift_codegen::{Context, FinalizedMachReloc};
 use cranelift_module::{
     DataDescription, DataId, FuncId, FuncOrDataId, Linkage, Module, ModuleDeclarations,
-    ModuleResult,
+    ModuleReloc, ModuleResult,
 };
 use cranelift_object::{ObjectModule, ObjectProduct};
 
@@ -17,8 +17,8 @@ pub(crate) struct UnwindModule<T> {
 }
 
 impl<T: Module> UnwindModule<T> {
-    pub(crate) fn new(module: T, pic_eh_frame: bool) -> Self {
-        let unwind_context = UnwindContext::new(module.isa(), pic_eh_frame);
+    pub(crate) fn new(mut module: T, pic_eh_frame: bool) -> Self {
+        let unwind_context = UnwindContext::new(&mut module, pic_eh_frame);
         UnwindModule { module, unwind_context }
     }
 }
@@ -33,13 +33,10 @@ impl UnwindModule<ObjectModule> {
 
 #[cfg(feature = "jit")]
 impl UnwindModule<cranelift_jit::JITModule> {
-    pub(crate) fn finalize_definitions(&mut self) {
+    pub(crate) fn finalize_definitions(mut self) -> cranelift_jit::JITModule {
         self.module.finalize_definitions().unwrap();
-        let prev_unwind_context = std::mem::replace(
-            &mut self.unwind_context,
-            UnwindContext::new(self.module.isa(), false),
-        );
-        unsafe { prev_unwind_context.register_jit(&self.module) };
+        unsafe { self.unwind_context.register_jit(&self.module) };
+        self.module
     }
 }
 
@@ -94,17 +91,16 @@ impl<T: Module> Module for UnwindModule<T> {
         ctrl_plane: &mut ControlPlane,
     ) -> ModuleResult<()> {
         self.module.define_function_with_control_plane(func, ctx, ctrl_plane)?;
-        self.unwind_context.add_function(func, ctx, self.module.isa());
+        self.unwind_context.add_function(&mut self.module, func, ctx);
         Ok(())
     }
 
     fn define_function_bytes(
         &mut self,
         _func_id: FuncId,
-        _func: &Function,
         _alignment: u64,
         _bytes: &[u8],
-        _relocs: &[FinalizedMachReloc],
+        _relocs: &[ModuleReloc],
     ) -> ModuleResult<()> {
         unimplemented!()
     }
diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
index f8a19589fdd..cbfb215a892 100644
--- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs
+++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
@@ -309,6 +309,7 @@ impl<'tcx> CValue<'tcx> {
         match self.0 {
             CValueInner::ByVal(_) | CValueInner::ByValPair(_, _) => unreachable!(),
             CValueInner::ByRef(ptr, None) => {
+                let lane_idx = clif_intcast(fx, lane_idx, fx.pointer_type, false);
                 let field_offset = fx.bcx.ins().imul_imm(lane_idx, lane_layout.size.bytes() as i64);
                 let field_ptr = ptr.offset_value(fx, field_offset);
                 CValue::by_ref(field_ptr, lane_layout)
@@ -324,7 +325,7 @@ impl<'tcx> CValue<'tcx> {
         const_val: ty::ScalarInt,
     ) -> CValue<'tcx> {
         assert_eq!(const_val.size(), layout.size, "{:#?}: {:?}", const_val, layout);
-        use cranelift_codegen::ir::immediates::{Ieee32, Ieee64};
+        use cranelift_codegen::ir::immediates::{Ieee16, Ieee32, Ieee64, Ieee128};
 
         let clif_ty = fx.clif_type(layout.ty).unwrap();
 
@@ -345,12 +346,24 @@ impl<'tcx> CValue<'tcx> {
                 let raw_val = const_val.size().truncate(const_val.to_bits(layout.size));
                 fx.bcx.ins().iconst(clif_ty, raw_val as i64)
             }
+            ty::Float(FloatTy::F16) => {
+                fx.bcx.ins().f16const(Ieee16::with_bits(u16::try_from(const_val).unwrap()))
+            }
             ty::Float(FloatTy::F32) => {
                 fx.bcx.ins().f32const(Ieee32::with_bits(u32::try_from(const_val).unwrap()))
             }
             ty::Float(FloatTy::F64) => {
                 fx.bcx.ins().f64const(Ieee64::with_bits(u64::try_from(const_val).unwrap()))
             }
+            ty::Float(FloatTy::F128) => {
+                let value = fx
+                    .bcx
+                    .func
+                    .dfg
+                    .constants
+                    .insert(Ieee128::with_bits(u128::try_from(const_val).unwrap()).into());
+                fx.bcx.ins().f128const(value)
+            }
             _ => panic!(
                 "CValue::const_val for non bool/char/float/integer/pointer type {:?} is not allowed",
                 layout.ty
@@ -563,27 +576,7 @@ impl<'tcx> CPlace<'tcx> {
                 src_ty,
                 dst_ty,
             );
-            let data = match (src_ty, dst_ty) {
-                (_, _) if src_ty == dst_ty => data,
-
-                // This is a `write_cvalue_transmute`.
-                (types::I32, types::F32)
-                | (types::F32, types::I32)
-                | (types::I64, types::F64)
-                | (types::F64, types::I64) => codegen_bitcast(fx, dst_ty, data),
-                _ if src_ty.is_vector() && dst_ty.is_vector() => codegen_bitcast(fx, dst_ty, data),
-                _ if src_ty.is_vector() || dst_ty.is_vector() => {
-                    // FIXME(bytecodealliance/wasmtime#6104) do something more efficient for transmutes between vectors and integers.
-                    let ptr = fx.create_stack_slot(src_ty.bytes(), src_ty.bytes());
-                    ptr.store(fx, data, MemFlags::trusted());
-                    ptr.load(fx, dst_ty, MemFlags::trusted())
-                }
-
-                // `CValue`s should never contain SSA-only types, so if you ended
-                // up here having seen an error like `B1 -> I8`, then before
-                // calling `write_cvalue` you need to add a `bint` instruction.
-                _ => unreachable!("write_cvalue_transmute: {:?} -> {:?}", src_ty, dst_ty),
-            };
+            let data = if src_ty == dst_ty { data } else { codegen_bitcast(fx, dst_ty, data) };
             //fx.bcx.set_val_label(data, cranelift_codegen::ir::ValueLabel::new(var.index()));
             fx.bcx.def_var(var, data);
         }
@@ -591,13 +584,9 @@ impl<'tcx> CPlace<'tcx> {
         assert_eq!(self.layout().size, from.layout().size);
 
         if fx.clif_comments.enabled() {
-            use cranelift_codegen::cursor::{Cursor, CursorPosition};
-            let cur_block = match fx.bcx.cursor().position() {
-                CursorPosition::After(block) => block,
-                _ => unreachable!(),
-            };
-            fx.add_comment(
-                fx.bcx.func.layout.last_inst(cur_block).unwrap(),
+            let inst = fx.bcx.func.layout.last_inst(fx.bcx.current_block().unwrap()).unwrap();
+            fx.add_post_comment(
+                inst,
                 format!(
                     "{}: {:?}: {:?} <- {:?}: {:?}",
                     method,
@@ -806,6 +795,35 @@ impl<'tcx> CPlace<'tcx> {
         }
     }
 
+    /// Write a value to an individual lane in a SIMD vector.
+    pub(crate) fn write_lane_dyn(
+        self,
+        fx: &mut FunctionCx<'_, '_, 'tcx>,
+        lane_idx: Value,
+        value: CValue<'tcx>,
+    ) {
+        let layout = self.layout();
+        assert!(layout.ty.is_simd());
+        let (_lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+        let lane_layout = fx.layout_of(lane_ty);
+        assert_eq!(lane_layout, value.layout());
+
+        match self.inner {
+            CPlaceInner::Var(_, _) => unreachable!(),
+            CPlaceInner::VarPair(_, _, _) => unreachable!(),
+            CPlaceInner::Addr(ptr, None) => {
+                let lane_idx = clif_intcast(fx, lane_idx, fx.pointer_type, false);
+                let field_offset = fx
+                    .bcx
+                    .ins()
+                    .imul_imm(lane_idx, i64::try_from(lane_layout.size.bytes()).unwrap());
+                let field_ptr = ptr.offset_value(fx, field_offset);
+                CPlace::for_ptr(field_ptr, lane_layout).write_cvalue(fx, value);
+            }
+            CPlaceInner::Addr(_, Some(_)) => unreachable!(),
+        }
+    }
+
     pub(crate) fn place_index(
         self,
         fx: &mut FunctionCx<'_, '_, 'tcx>,
diff --git a/compiler/rustc_codegen_cranelift/src/vtable.rs b/compiler/rustc_codegen_cranelift/src/vtable.rs
index 9d9e0462a9b..05a8e3c3342 100644
--- a/compiler/rustc_codegen_cranelift/src/vtable.rs
+++ b/compiler/rustc_codegen_cranelift/src/vtable.rs
@@ -53,7 +53,7 @@ pub(crate) fn get_ptr_and_method_ref<'tcx>(
                     .layout()
                     .non_1zst_field(fx)
                     .expect("not exactly one non-1-ZST field in a `DispatchFromDyn` type");
-                arg = arg.value_field(fx, FieldIdx::new(idx));
+                arg = arg.value_field(fx, idx);
             }
         }
 
@@ -62,8 +62,7 @@ pub(crate) fn get_ptr_and_method_ref<'tcx>(
                 let inner_layout = fx.layout_of(arg.layout().ty.builtin_deref(true).unwrap());
                 let dyn_star = CPlace::for_ptr(Pointer::new(arg.load_scalar(fx)), inner_layout);
                 let ptr = dyn_star.place_field(fx, FieldIdx::ZERO).to_ptr();
-                let vtable =
-                    dyn_star.place_field(fx, FieldIdx::new(1)).to_cvalue(fx).load_scalar(fx);
+                let vtable = dyn_star.place_field(fx, FieldIdx::ONE).to_cvalue(fx).load_scalar(fx);
                 break 'block (ptr, vtable);
             }
         }
diff --git a/compiler/rustc_codegen_cranelift/triagebot.toml b/compiler/rustc_codegen_cranelift/triagebot.toml
new file mode 100644
index 00000000000..13da0a87def
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/triagebot.toml
@@ -0,0 +1,7 @@
+# Documentation at https://forge.rust-lang.org/triagebot/index.html
+
+# Prevents un-canonicalized issue links (to avoid wrong issues being linked in r-l/rust)
+[issue-links]
+
+# Prevents mentions in commits to avoid users being spammed
+[no-mentions]
diff --git a/compiler/rustc_codegen_gcc/example/alloc_system.rs b/compiler/rustc_codegen_gcc/example/alloc_system.rs
index 945d34063a6..4d70122496b 100644
--- a/compiler/rustc_codegen_gcc/example/alloc_system.rs
+++ b/compiler/rustc_codegen_gcc/example/alloc_system.rs
@@ -8,6 +8,7 @@
 // add fast paths for low alignment values.
 #[cfg(any(target_arch = "x86",
               target_arch = "arm",
+              target_arch = "loongarch32",
               target_arch = "m68k",
               target_arch = "mips",
               target_arch = "mips32r6",
diff --git a/compiler/rustc_codegen_gcc/messages.ftl b/compiler/rustc_codegen_gcc/messages.ftl
index 882fff8673a..546bfc87b68 100644
--- a/compiler/rustc_codegen_gcc/messages.ftl
+++ b/compiler/rustc_codegen_gcc/messages.ftl
@@ -2,9 +2,6 @@ codegen_gcc_unknown_ctarget_feature_prefix =
     unknown feature specified for `-Ctarget-feature`: `{$feature}`
     .note = features must begin with a `+` to enable or `-` to disable it
 
-codegen_gcc_invalid_minimum_alignment =
-    invalid minimum global alignment: {$err}
-
 codegen_gcc_forbidden_ctarget_feature =
     target feature `{$feature}` cannot be toggled with `-Ctarget-feature`: {$reason}
 
diff --git a/compiler/rustc_codegen_gcc/src/abi.rs b/compiler/rustc_codegen_gcc/src/abi.rs
index d882d3eecf4..3d0c258f576 100644
--- a/compiler/rustc_codegen_gcc/src/abi.rs
+++ b/compiler/rustc_codegen_gcc/src/abi.rs
@@ -1,7 +1,7 @@
 #[cfg(feature = "master")]
 use gccjit::FnAttribute;
 use gccjit::{ToLValue, ToRValue, Type};
-use rustc_abi::{Reg, RegKind};
+use rustc_abi::{ArmCall, CanonAbi, InterruptKind, Reg, RegKind, X86Call};
 use rustc_codegen_ssa::traits::{AbiBuilderMethods, BaseTypeCodegenMethods};
 use rustc_data_structures::fx::FxHashSet;
 use rustc_middle::bug;
@@ -10,12 +10,9 @@ use rustc_middle::ty::layout::LayoutOf;
 #[cfg(feature = "master")]
 use rustc_session::config;
 use rustc_target::callconv::{ArgAttributes, CastTarget, FnAbi, PassMode};
-#[cfg(feature = "master")]
-use rustc_target::callconv::{Conv, RiscvInterruptKind};
 
 use crate::builder::Builder;
 use crate::context::CodegenCx;
-use crate::intrinsic::ArgAbiExt;
 use crate::type_of::LayoutGccExt;
 
 impl AbiBuilderMethods for Builder<'_, '_, '_> {
@@ -125,7 +122,7 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_gcc_type(cx),
             PassMode::Cast { ref cast, .. } => cast.gcc_type(cx),
             PassMode::Indirect { .. } => {
-                argument_tys.push(cx.type_ptr_to(self.ret.memory_ty(cx)));
+                argument_tys.push(cx.type_ptr_to(self.ret.layout.gcc_type(cx)));
                 cx.type_void()
             }
         };
@@ -176,13 +173,13 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                 PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: true } => {
                     // This is a "byval" argument, so we don't apply the `restrict` attribute on it.
                     on_stack_param_indices.insert(argument_tys.len());
-                    arg.memory_ty(cx)
+                    arg.layout.gcc_type(cx)
                 }
                 PassMode::Direct(attrs) => {
                     apply_attrs(arg.layout.immediate_gcc_type(cx), &attrs, argument_tys.len())
                 }
                 PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => {
-                    apply_attrs(cx.type_ptr_to(arg.memory_ty(cx)), &attrs, argument_tys.len())
+                    apply_attrs(cx.type_ptr_to(arg.layout.gcc_type(cx)), &attrs, argument_tys.len())
                 }
                 PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => {
                     assert!(!on_stack);
@@ -239,29 +236,16 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
 }
 
 #[cfg(feature = "master")]
-pub fn conv_to_fn_attribute<'gcc>(conv: Conv, arch: &str) -> Option<FnAttribute<'gcc>> {
+pub fn conv_to_fn_attribute<'gcc>(conv: CanonAbi, arch: &str) -> Option<FnAttribute<'gcc>> {
     let attribute = match conv {
-        Conv::C | Conv::Rust => return None,
-        Conv::CCmseNonSecureCall => {
-            if arch == "arm" {
-                FnAttribute::ArmCmseNonsecureCall
-            } else {
-                return None;
-            }
-        }
-        Conv::CCmseNonSecureEntry => {
-            if arch == "arm" {
-                FnAttribute::ArmCmseNonsecureEntry
-            } else {
-                return None;
-            }
-        }
-        Conv::Cold => FnAttribute::Cold,
-        // NOTE: the preserve attributes are not yet implemented in GCC:
-        // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=110899
-        Conv::PreserveMost => return None,
-        Conv::PreserveAll => return None,
-        Conv::GpuKernel => {
+        CanonAbi::C | CanonAbi::Rust => return None,
+        CanonAbi::Arm(arm_call) => match arm_call {
+            ArmCall::CCmseNonSecureCall => FnAttribute::ArmCmseNonsecureCall,
+            ArmCall::CCmseNonSecureEntry => FnAttribute::ArmCmseNonsecureEntry,
+            ArmCall::Aapcs => FnAttribute::ArmPcs("aapcs"),
+        },
+        CanonAbi::RustCold => FnAttribute::Cold,
+        CanonAbi::GpuKernel => {
             if arch == "amdgpu" {
                 FnAttribute::GcnAmdGpuHsaKernel
             } else if arch == "nvptx64" {
@@ -271,26 +255,24 @@ pub fn conv_to_fn_attribute<'gcc>(conv: Conv, arch: &str) -> Option<FnAttribute<
             }
         }
         // TODO(antoyo): check if those AVR attributes are mapped correctly.
-        Conv::AvrInterrupt => FnAttribute::AvrSignal,
-        Conv::AvrNonBlockingInterrupt => FnAttribute::AvrInterrupt,
-        Conv::ArmAapcs => FnAttribute::ArmPcs("aapcs"),
-        Conv::Msp430Intr => FnAttribute::Msp430Interrupt,
-        Conv::RiscvInterrupt { kind } => {
-            let kind = match kind {
-                RiscvInterruptKind::Machine => "machine",
-                RiscvInterruptKind::Supervisor => "supervisor",
-            };
-            FnAttribute::RiscvInterrupt(kind)
-        }
-        Conv::X86Fastcall => FnAttribute::X86FastCall,
-        Conv::X86Intr => FnAttribute::X86Interrupt,
-        Conv::X86Stdcall => FnAttribute::X86Stdcall,
-        Conv::X86ThisCall => FnAttribute::X86ThisCall,
-        // NOTE: the vectorcall calling convention is not yet implemented in GCC:
-        // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89485
-        Conv::X86VectorCall => return None,
-        Conv::X86_64SysV => FnAttribute::X86SysvAbi,
-        Conv::X86_64Win64 => FnAttribute::X86MsAbi,
+        CanonAbi::Interrupt(interrupt_kind) => match interrupt_kind {
+            InterruptKind::Avr => FnAttribute::AvrSignal,
+            InterruptKind::AvrNonBlocking => FnAttribute::AvrInterrupt,
+            InterruptKind::Msp430 => FnAttribute::Msp430Interrupt,
+            InterruptKind::RiscvMachine => FnAttribute::RiscvInterrupt("machine"),
+            InterruptKind::RiscvSupervisor => FnAttribute::RiscvInterrupt("supervisor"),
+            InterruptKind::X86 => FnAttribute::X86Interrupt,
+        },
+        CanonAbi::X86(x86_call) => match x86_call {
+            X86Call::Fastcall => FnAttribute::X86FastCall,
+            X86Call::Stdcall => FnAttribute::X86Stdcall,
+            X86Call::Thiscall => FnAttribute::X86ThisCall,
+            // // NOTE: the vectorcall calling convention is not yet implemented in GCC:
+            // // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89485
+            X86Call::Vectorcall => return None,
+            X86Call::SysV64 => FnAttribute::X86SysvAbi,
+            X86Call::Win64 => FnAttribute::X86MsAbi,
+        },
     };
     Some(attribute)
 }
diff --git a/compiler/rustc_codegen_gcc/src/base.rs b/compiler/rustc_codegen_gcc/src/base.rs
index a9d7808c833..c105916bbb2 100644
--- a/compiler/rustc_codegen_gcc/src/base.rs
+++ b/compiler/rustc_codegen_gcc/src/base.rs
@@ -219,17 +219,22 @@ pub fn compile_codegen_unit(
 
             let mono_items = cgu.items_in_deterministic_order(tcx);
             for &(mono_item, data) in &mono_items {
-                mono_item.predefine::<Builder<'_, '_, '_>>(&cx, data.linkage, data.visibility);
+                mono_item.predefine::<Builder<'_, '_, '_>>(
+                    &mut cx,
+                    cgu_name.as_str(),
+                    data.linkage,
+                    data.visibility,
+                );
             }
 
             // ... and now that we have everything pre-defined, fill out those definitions.
             for &(mono_item, item_data) in &mono_items {
-                mono_item.define::<Builder<'_, '_, '_>>(&mut cx, item_data);
+                mono_item.define::<Builder<'_, '_, '_>>(&mut cx, cgu_name.as_str(), item_data);
             }
 
             // If this codegen unit contains the main function, also create the
             // wrapper here
-            maybe_create_entry_wrapper::<Builder<'_, '_, '_>>(&cx);
+            maybe_create_entry_wrapper::<Builder<'_, '_, '_>>(&cx, cx.codegen_unit);
 
             // Finalize debuginfo
             if cx.sess().opts.debuginfo != DebugInfo::None {
diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs
index 4e2163201fd..d1fb8d8f9d6 100644
--- a/compiler/rustc_codegen_gcc/src/builder.rs
+++ b/compiler/rustc_codegen_gcc/src/builder.rs
@@ -12,7 +12,7 @@ use rustc_abi::{Align, HasDataLayout, Size, TargetDataLayout, WrappingRange};
 use rustc_apfloat::{Float, Round, Status, ieee};
 use rustc_codegen_ssa::MemFlags;
 use rustc_codegen_ssa::common::{
-    AtomicOrdering, AtomicRmwBinOp, IntPredicate, RealPredicate, SynchronizationScope, TypeKind,
+    AtomicRmwBinOp, IntPredicate, RealPredicate, SynchronizationScope, TypeKind,
 };
 use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
 use rustc_codegen_ssa::mir::place::PlaceRef;
@@ -26,7 +26,7 @@ use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
 use rustc_middle::ty::layout::{
     FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasTyCtxt, HasTypingEnv, LayoutError, LayoutOfHelpers,
 };
-use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
+use rustc_middle::ty::{self, AtomicOrdering, Instance, Ty, TyCtxt};
 use rustc_span::Span;
 use rustc_span::def_id::DefId;
 use rustc_target::callconv::FnAbi;
@@ -75,7 +75,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
 
         let load_ordering = match order {
             // TODO(antoyo): does this make sense?
-            AtomicOrdering::AcquireRelease | AtomicOrdering::Release => AtomicOrdering::Acquire,
+            AtomicOrdering::AcqRel | AtomicOrdering::Release => AtomicOrdering::Acquire,
             _ => order,
         };
         let previous_value =
@@ -2474,8 +2474,8 @@ impl ToGccOrdering for AtomicOrdering {
             AtomicOrdering::Relaxed => __ATOMIC_RELAXED, // TODO(antoyo): check if that's the same.
             AtomicOrdering::Acquire => __ATOMIC_ACQUIRE,
             AtomicOrdering::Release => __ATOMIC_RELEASE,
-            AtomicOrdering::AcquireRelease => __ATOMIC_ACQ_REL,
-            AtomicOrdering::SequentiallyConsistent => __ATOMIC_SEQ_CST,
+            AtomicOrdering::AcqRel => __ATOMIC_ACQ_REL,
+            AtomicOrdering::SeqCst => __ATOMIC_SEQ_CST,
         };
         ordering as i32
     }
diff --git a/compiler/rustc_codegen_gcc/src/consts.rs b/compiler/rustc_codegen_gcc/src/consts.rs
index 033afc0f8fb..1690641a5bc 100644
--- a/compiler/rustc_codegen_gcc/src/consts.rs
+++ b/compiler/rustc_codegen_gcc/src/consts.rs
@@ -18,7 +18,6 @@ use rustc_span::def_id::DefId;
 
 use crate::base;
 use crate::context::CodegenCx;
-use crate::errors::InvalidMinimumAlignment;
 use crate::type_of::LayoutGccExt;
 
 fn set_global_alignment<'gcc, 'tcx>(
@@ -29,13 +28,8 @@ fn set_global_alignment<'gcc, 'tcx>(
     // The target may require greater alignment for globals than the type does.
     // Note: GCC and Clang also allow `__attribute__((aligned))` on variables,
     // which can force it to be smaller. Rust doesn't support this yet.
-    if let Some(min) = cx.sess().target.min_global_align {
-        match Align::from_bits(min) {
-            Ok(min) => align = align.max(min),
-            Err(err) => {
-                cx.sess().dcx().emit_err(InvalidMinimumAlignment { err: err.to_string() });
-            }
-        }
+    if let Some(min_global) = cx.sess().target.min_global_align {
+        align = Ord::max(align, min_global);
     }
     gv.set_alignment(align.bytes() as i32);
 }
@@ -67,7 +61,7 @@ impl<'gcc, 'tcx> StaticCodegenMethods for CodegenCx<'gcc, 'tcx> {
     }
 
     #[cfg_attr(not(feature = "master"), allow(unused_mut))]
-    fn codegen_static(&self, def_id: DefId) {
+    fn codegen_static(&mut self, def_id: DefId) {
         let attrs = self.tcx.codegen_fn_attrs(def_id);
 
         let Ok((value, alloc)) = codegen_static_initializer(self, def_id) else {
@@ -154,25 +148,20 @@ impl<'gcc, 'tcx> StaticCodegenMethods for CodegenCx<'gcc, 'tcx> {
             // TODO(antoyo): set link section.
         }
 
-        if attrs.flags.contains(CodegenFnAttrFlags::USED)
+        if attrs.flags.contains(CodegenFnAttrFlags::USED_COMPILER)
             || attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER)
         {
             self.add_used_global(global.to_rvalue());
         }
     }
+}
 
+impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
     /// Add a global value to a list to be stored in the `llvm.used` variable, an array of i8*.
-    fn add_used_global(&self, _global: RValue<'gcc>) {
+    pub fn add_used_global(&mut self, _global: RValue<'gcc>) {
         // TODO(antoyo)
     }
 
-    fn add_compiler_used_global(&self, global: RValue<'gcc>) {
-        // NOTE: seems like GCC does not make the distinction between compiler.used and used.
-        self.add_used_global(global);
-    }
-}
-
-impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
     #[cfg_attr(not(feature = "master"), allow(unused_variables))]
     pub fn add_used_function(&self, function: Function<'gcc>) {
         #[cfg(feature = "master")]
diff --git a/compiler/rustc_codegen_gcc/src/context.rs b/compiler/rustc_codegen_gcc/src/context.rs
index 73718994e64..c6c43201f21 100644
--- a/compiler/rustc_codegen_gcc/src/context.rs
+++ b/compiler/rustc_codegen_gcc/src/context.rs
@@ -470,10 +470,6 @@ impl<'gcc, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
         self.tcx.sess
     }
 
-    fn codegen_unit(&self) -> &'tcx CodegenUnit<'tcx> {
-        self.codegen_unit
-    }
-
     fn set_frame_pointer_type(&self, _llfn: RValue<'gcc>) {
         // TODO(antoyo)
     }
diff --git a/compiler/rustc_codegen_gcc/src/debuginfo.rs b/compiler/rustc_codegen_gcc/src/debuginfo.rs
index e0597d0030d..3a265fbc64f 100644
--- a/compiler/rustc_codegen_gcc/src/debuginfo.rs
+++ b/compiler/rustc_codegen_gcc/src/debuginfo.rs
@@ -52,10 +52,6 @@ impl<'a, 'gcc, 'tcx> DebugInfoBuilderMethods for Builder<'a, 'gcc, 'tcx> {
     fn clear_dbg_loc(&mut self) {
         self.location = None;
     }
-
-    fn get_dbg_loc(&self) -> Option<Self::DILocation> {
-        self.location
-    }
 }
 
 /// Generate the `debug_context` in an MIR Body.
diff --git a/compiler/rustc_codegen_gcc/src/errors.rs b/compiler/rustc_codegen_gcc/src/errors.rs
index 1b59b9ac169..ccd9abe3804 100644
--- a/compiler/rustc_codegen_gcc/src/errors.rs
+++ b/compiler/rustc_codegen_gcc/src/errors.rs
@@ -48,12 +48,6 @@ pub(crate) struct UnwindingInlineAsm {
 }
 
 #[derive(Diagnostic)]
-#[diag(codegen_gcc_invalid_minimum_alignment)]
-pub(crate) struct InvalidMinimumAlignment {
-    pub err: String,
-}
-
-#[derive(Diagnostic)]
 #[diag(codegen_gcc_copy_bitcode)]
 pub(crate) struct CopyBitcode {
     pub err: std::io::Error,
diff --git a/compiler/rustc_codegen_gcc/src/int.rs b/compiler/rustc_codegen_gcc/src/int.rs
index 9b5b0fde6e2..eb4acd8ade9 100644
--- a/compiler/rustc_codegen_gcc/src/int.rs
+++ b/compiler/rustc_codegen_gcc/src/int.rs
@@ -3,11 +3,11 @@
 //! 128-bit integers on 32-bit platforms and thus require to be handled manually.
 
 use gccjit::{BinaryOp, ComparisonOp, FunctionType, Location, RValue, ToRValue, Type, UnaryOp};
-use rustc_abi::{Endian, ExternAbi};
+use rustc_abi::{CanonAbi, Endian, ExternAbi};
 use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
 use rustc_codegen_ssa::traits::{BackendTypes, BaseTypeCodegenMethods, BuilderMethods, OverflowOp};
 use rustc_middle::ty::{self, Ty};
-use rustc_target::callconv::{ArgAbi, ArgAttributes, Conv, FnAbi, PassMode};
+use rustc_target::callconv::{ArgAbi, ArgAttributes, FnAbi, PassMode};
 
 use crate::builder::{Builder, ToGccComp};
 use crate::common::{SignType, TypeReflection};
@@ -397,7 +397,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
             ret: arg_abi,
             c_variadic: false,
             fixed_count: 3,
-            conv: Conv::C,
+            conv: CanonAbi::C,
             can_unwind: false,
         };
         fn_abi.adjust_for_foreign_abi(self.cx, ExternAbi::C { unwind: false });
diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
index ba65c8205a5..9e05b8f23aa 100644
--- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
+++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
@@ -22,11 +22,11 @@ use rustc_codegen_ssa::traits::{
 };
 use rustc_middle::bug;
 #[cfg(feature = "master")]
-use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt};
-use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf};
+use rustc_middle::ty::layout::FnAbiOf;
+use rustc_middle::ty::layout::LayoutOf;
 use rustc_middle::ty::{self, Instance, Ty};
 use rustc_span::{Span, Symbol, sym};
-use rustc_target::callconv::{ArgAbi, FnAbi, PassMode};
+use rustc_target::callconv::{ArgAbi, PassMode};
 use rustc_target::spec::PanicStrategy;
 
 #[cfg(feature = "master")]
@@ -200,28 +200,15 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
     fn codegen_intrinsic_call(
         &mut self,
         instance: Instance<'tcx>,
-        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
         args: &[OperandRef<'tcx, RValue<'gcc>>],
-        llresult: RValue<'gcc>,
+        result: PlaceRef<'tcx, RValue<'gcc>>,
         span: Span,
     ) -> Result<(), Instance<'tcx>> {
         let tcx = self.tcx;
-        let callee_ty = instance.ty(tcx, self.typing_env());
 
-        let (def_id, fn_args) = match *callee_ty.kind() {
-            ty::FnDef(def_id, fn_args) => (def_id, fn_args),
-            _ => bug!("expected fn item type, found {}", callee_ty),
-        };
-
-        let sig = callee_ty.fn_sig(tcx);
-        let sig = tcx.normalize_erasing_late_bound_regions(self.typing_env(), sig);
-        let arg_tys = sig.inputs();
-        let ret_ty = sig.output();
-        let name = tcx.item_name(def_id);
+        let name = tcx.item_name(instance.def_id());
         let name_str = name.as_str();
-
-        let llret_ty = self.layout_of(ret_ty).gcc_type(self);
-        let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout);
+        let fn_args = instance.args;
 
         let simple = get_simple_intrinsic(self, name);
         let simple_func = get_simple_function(self, name);
@@ -271,7 +258,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
                     args[0].immediate(),
                     args[1].immediate(),
                     args[2].immediate(),
-                    llresult,
+                    result,
                 );
                 return Ok(());
             }
@@ -286,17 +273,10 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
             }
 
             sym::volatile_load | sym::unaligned_volatile_load => {
-                let tp_ty = fn_args.type_at(0);
                 let ptr = args[0].immediate();
-                let layout = self.layout_of(tp_ty);
-                let load = if let PassMode::Cast { cast: ref ty, pad_i32: _ } = fn_abi.ret.mode {
-                    let gcc_ty = ty.gcc_type(self);
-                    self.volatile_load(gcc_ty, ptr)
-                } else {
-                    self.volatile_load(layout.gcc_type(self), ptr)
-                };
+                let load = self.volatile_load(result.layout.gcc_type(self), ptr);
                 // TODO(antoyo): set alignment.
-                if let BackendRepr::Scalar(scalar) = layout.backend_repr {
+                if let BackendRepr::Scalar(scalar) = result.layout.backend_repr {
                     self.to_immediate_scalar(load, scalar)
                 } else {
                     load
@@ -329,8 +309,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
             | sym::rotate_right
             | sym::saturating_add
             | sym::saturating_sub => {
-                let ty = arg_tys[0];
-                match int_type_width_signed(ty, self) {
+                match int_type_width_signed(args[0].layout.ty, self) {
                     Some((width, signed)) => match name {
                         sym::ctlz | sym::cttz => {
                             let func = self.current_func.borrow().expect("func");
@@ -409,7 +388,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
                         tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
                             span,
                             name,
-                            ty,
+                            ty: args[0].layout.ty,
                         });
                         return Ok(());
                     }
@@ -501,7 +480,14 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
             }
 
             _ if name_str.starts_with("simd_") => {
-                match generic_simd_intrinsic(self, name, callee_ty, args, ret_ty, llret_ty, span) {
+                match generic_simd_intrinsic(
+                    self,
+                    name,
+                    args,
+                    result.layout.ty,
+                    result.layout.gcc_type(self),
+                    span,
+                ) {
                     Ok(value) => value,
                     Err(()) => return Ok(()),
                 }
@@ -511,16 +497,11 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
             _ => return Err(Instance::new_raw(instance.def_id(), instance.args)),
         };
 
-        if !fn_abi.ret.is_ignore() {
-            if let PassMode::Cast { cast: ref ty, .. } = fn_abi.ret.mode {
-                let ptr_llty = self.type_ptr_to(ty.gcc_type(self));
-                let ptr = self.pointercast(result.val.llval, ptr_llty);
-                self.store(value, ptr, result.val.align);
-            } else {
-                OperandRef::from_immediate_or_packed_pair(self, value, result.layout)
-                    .val
-                    .store(self, result);
-            }
+        if result.layout.ty.is_bool() {
+            let val = self.from_immediate(value);
+            self.store_to_place(val, result.val);
+        } else if !result.layout.ty.is_unit() {
+            self.store_to_place(value, result.val);
         }
         Ok(())
     }
@@ -543,11 +524,6 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
         cond
     }
 
-    fn type_test(&mut self, _pointer: Self::Value, _typeid: Self::Value) -> Self::Value {
-        // Unsupported.
-        self.context.new_rvalue_from_int(self.int_type, 0)
-    }
-
     fn type_checked_load(
         &mut self,
         _llvtable: Self::Value,
@@ -585,14 +561,9 @@ impl<'a, 'gcc, 'tcx> ArgAbiBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
     ) {
         arg_abi.store(self, val, dst)
     }
-
-    fn arg_memory_ty(&self, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>) -> Type<'gcc> {
-        arg_abi.memory_ty(self)
-    }
 }
 
 pub trait ArgAbiExt<'gcc, 'tcx> {
-    fn memory_ty(&self, cx: &CodegenCx<'gcc, 'tcx>) -> Type<'gcc>;
     fn store(
         &self,
         bx: &mut Builder<'_, 'gcc, 'tcx>,
@@ -608,12 +579,6 @@ pub trait ArgAbiExt<'gcc, 'tcx> {
 }
 
 impl<'gcc, 'tcx> ArgAbiExt<'gcc, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
-    /// Gets the LLVM type for a place of the original Rust type of
-    /// this argument/return, i.e., the result of `type_of::type_of`.
-    fn memory_ty(&self, cx: &CodegenCx<'gcc, 'tcx>) -> Type<'gcc> {
-        self.layout.gcc_type(cx)
-    }
-
     /// Stores a direct/indirect value described by this ArgAbi into a
     /// place for the original Rust type of this argument/return.
     /// Can be used for both storing formal arguments into Rust variables
@@ -1230,14 +1195,13 @@ fn try_intrinsic<'a, 'b, 'gcc, 'tcx>(
     try_func: RValue<'gcc>,
     data: RValue<'gcc>,
     _catch_func: RValue<'gcc>,
-    dest: RValue<'gcc>,
+    dest: PlaceRef<'tcx, RValue<'gcc>>,
 ) {
     if bx.sess().panic_strategy() == PanicStrategy::Abort {
         bx.call(bx.type_void(), None, None, try_func, &[data], None, None);
         // Return 0 unconditionally from the intrinsic call;
         // we can never unwind.
-        let ret_align = bx.tcx.data_layout.i32_align.abi;
-        bx.store(bx.const_i32(0), dest, ret_align);
+        OperandValue::Immediate(bx.const_i32(0)).store(bx, dest);
     } else {
         if wants_msvc_seh(bx.sess()) {
             unimplemented!();
@@ -1261,12 +1225,12 @@ fn try_intrinsic<'a, 'b, 'gcc, 'tcx>(
 // functions in play. By calling a shim we're guaranteed that our shim will have
 // the right personality function.
 #[cfg(feature = "master")]
-fn codegen_gnu_try<'gcc>(
-    bx: &mut Builder<'_, 'gcc, '_>,
+fn codegen_gnu_try<'gcc, 'tcx>(
+    bx: &mut Builder<'_, 'gcc, 'tcx>,
     try_func: RValue<'gcc>,
     data: RValue<'gcc>,
     catch_func: RValue<'gcc>,
-    dest: RValue<'gcc>,
+    dest: PlaceRef<'tcx, RValue<'gcc>>,
 ) {
     let cx: &CodegenCx<'gcc, '_> = bx.cx;
     let (llty, func) = get_rust_try_fn(cx, &mut |mut bx| {
@@ -1322,8 +1286,7 @@ fn codegen_gnu_try<'gcc>(
     // Note that no invoke is used here because by definition this function
     // can't panic (that's what it's catching).
     let ret = bx.call(llty, None, None, func, &[try_func, data, catch_func], None, None);
-    let i32_align = bx.tcx().data_layout.i32_align.abi;
-    bx.store(ret, dest, i32_align);
+    OperandValue::Immediate(ret).store(bx, dest);
 }
 
 // Helper function used to get a handle to the `__rust_try` function used to
diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs b/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs
index b897d079249..82ef0d0b13a 100644
--- a/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs
+++ b/compiler/rustc_codegen_gcc/src/intrinsic/simd.rs
@@ -28,7 +28,6 @@ use crate::context::CodegenCx;
 pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
     bx: &mut Builder<'a, 'gcc, 'tcx>,
     name: Symbol,
-    callee_ty: Ty<'tcx>,
     args: &[OperandRef<'tcx, RValue<'gcc>>],
     ret_ty: Ty<'tcx>,
     llret_ty: Type<'gcc>,
@@ -54,24 +53,17 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         };
     }
 
-    let tcx = bx.tcx();
-    let sig = tcx.normalize_erasing_late_bound_regions(
-        ty::TypingEnv::fully_monomorphized(),
-        callee_ty.fn_sig(tcx),
-    );
-    let arg_tys = sig.inputs();
-
     if name == sym::simd_select_bitmask {
         require_simd!(
-            arg_tys[1],
-            InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] }
+            args[1].layout.ty,
+            InvalidMonomorphization::SimdArgument { span, name, ty: args[1].layout.ty }
         );
-        let (len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
+        let (len, _) = args[1].layout.ty.simd_size_and_type(bx.tcx());
 
         let expected_int_bits = (len.max(8) - 1).next_power_of_two();
         let expected_bytes = len / 8 + ((len % 8 > 0) as u64);
 
-        let mask_ty = arg_tys[0];
+        let mask_ty = args[0].layout.ty;
         let mut mask = match *mask_ty.kind() {
             ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
             ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
@@ -121,8 +113,11 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
     }
 
     // every intrinsic below takes a SIMD vector as its first argument
-    require_simd!(arg_tys[0], InvalidMonomorphization::SimdInput { span, name, ty: arg_tys[0] });
-    let in_ty = arg_tys[0];
+    require_simd!(
+        args[0].layout.ty,
+        InvalidMonomorphization::SimdInput { span, name, ty: args[0].layout.ty }
+    );
+    let in_ty = args[0].layout.ty;
 
     let comparison = match name {
         sym::simd_eq => Some(BinOp::Eq),
@@ -134,7 +129,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         _ => None,
     };
 
-    let (in_len, in_elem) = arg_tys[0].simd_size_and_type(bx.tcx());
+    let (in_len, in_elem) = args[0].layout.ty.simd_size_and_type(bx.tcx());
     if let Some(cmp_op) = comparison {
         require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
 
@@ -401,13 +396,13 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
     #[cfg(feature = "master")]
     if name == sym::simd_insert || name == sym::simd_insert_dyn {
         require!(
-            in_elem == arg_tys[2],
+            in_elem == args[2].layout.ty,
             InvalidMonomorphization::InsertedType {
                 span,
                 name,
                 in_elem,
                 in_ty,
-                out_ty: arg_tys[2]
+                out_ty: args[2].layout.ty
             }
         );
 
@@ -439,10 +434,10 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         let m_elem_ty = in_elem;
         let m_len = in_len;
         require_simd!(
-            arg_tys[1],
-            InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] }
+            args[1].layout.ty,
+            InvalidMonomorphization::SimdArgument { span, name, ty: args[1].layout.ty }
         );
-        let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
+        let (v_len, _) = args[1].layout.ty.simd_size_and_type(bx.tcx());
         require!(
             m_len == v_len,
             InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
@@ -911,18 +906,18 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         // All types must be simd vector types
         require_simd!(in_ty, InvalidMonomorphization::SimdFirst { span, name, ty: in_ty });
         require_simd!(
-            arg_tys[1],
-            InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] }
+            args[1].layout.ty,
+            InvalidMonomorphization::SimdSecond { span, name, ty: args[1].layout.ty }
         );
         require_simd!(
-            arg_tys[2],
-            InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] }
+            args[2].layout.ty,
+            InvalidMonomorphization::SimdThird { span, name, ty: args[2].layout.ty }
         );
         require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
 
         // Of the same length:
-        let (out_len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
-        let (out_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx());
+        let (out_len, _) = args[1].layout.ty.simd_size_and_type(bx.tcx());
+        let (out_len2, _) = args[2].layout.ty.simd_size_and_type(bx.tcx());
         require!(
             in_len == out_len,
             InvalidMonomorphization::SecondArgumentLength {
@@ -930,7 +925,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
                 name,
                 in_len,
                 in_ty,
-                arg_ty: arg_tys[1],
+                arg_ty: args[1].layout.ty,
                 out_len
             }
         );
@@ -941,7 +936,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
                 name,
                 in_len,
                 in_ty,
-                arg_ty: arg_tys[2],
+                arg_ty: args[2].layout.ty,
                 out_len: out_len2
             }
         );
@@ -970,8 +965,8 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
 
         // The second argument must be a simd vector with an element type that's a pointer
         // to the element type of the first argument
-        let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx());
-        let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx());
+        let (_, element_ty0) = args[0].layout.ty.simd_size_and_type(bx.tcx());
+        let (_, element_ty1) = args[1].layout.ty.simd_size_and_type(bx.tcx());
         let (pointer_count, underlying_ty) = match *element_ty1.kind() {
             ty::RawPtr(p_ty, _) if p_ty == in_elem => {
                 (ptr_count(element_ty1), non_ptr(element_ty1))
@@ -983,7 +978,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
                         span,
                         name,
                         expected_element: element_ty1,
-                        second_arg: arg_tys[1],
+                        second_arg: args[1].layout.ty,
                         in_elem,
                         in_ty,
                         mutability: ExpectedPointerMutability::Not,
@@ -998,7 +993,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
 
         // The element type of the third argument must be an integer type of any width:
         // TODO: also support unsigned integers.
-        let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx());
+        let (_, element_ty2) = args[2].layout.ty.simd_size_and_type(bx.tcx());
         match *element_ty2.kind() {
             ty::Int(_) => (),
             _ => {
@@ -1030,17 +1025,17 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         // All types must be simd vector types
         require_simd!(in_ty, InvalidMonomorphization::SimdFirst { span, name, ty: in_ty });
         require_simd!(
-            arg_tys[1],
-            InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] }
+            args[1].layout.ty,
+            InvalidMonomorphization::SimdSecond { span, name, ty: args[1].layout.ty }
         );
         require_simd!(
-            arg_tys[2],
-            InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] }
+            args[2].layout.ty,
+            InvalidMonomorphization::SimdThird { span, name, ty: args[2].layout.ty }
         );
 
         // Of the same length:
-        let (element_len1, _) = arg_tys[1].simd_size_and_type(bx.tcx());
-        let (element_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx());
+        let (element_len1, _) = args[1].layout.ty.simd_size_and_type(bx.tcx());
+        let (element_len2, _) = args[2].layout.ty.simd_size_and_type(bx.tcx());
         require!(
             in_len == element_len1,
             InvalidMonomorphization::SecondArgumentLength {
@@ -1048,7 +1043,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
                 name,
                 in_len,
                 in_ty,
-                arg_ty: arg_tys[1],
+                arg_ty: args[1].layout.ty,
                 out_len: element_len1
             }
         );
@@ -1059,7 +1054,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
                 name,
                 in_len,
                 in_ty,
-                arg_ty: arg_tys[2],
+                arg_ty: args[2].layout.ty,
                 out_len: element_len2
             }
         );
@@ -1082,9 +1077,9 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
 
         // The second argument must be a simd vector with an element type that's a pointer
         // to the element type of the first argument
-        let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx());
-        let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx());
-        let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx());
+        let (_, element_ty0) = args[0].layout.ty.simd_size_and_type(bx.tcx());
+        let (_, element_ty1) = args[1].layout.ty.simd_size_and_type(bx.tcx());
+        let (_, element_ty2) = args[2].layout.ty.simd_size_and_type(bx.tcx());
         let (pointer_count, underlying_ty) = match *element_ty1.kind() {
             ty::RawPtr(p_ty, mutbl) if p_ty == in_elem && mutbl == hir::Mutability::Mut => {
                 (ptr_count(element_ty1), non_ptr(element_ty1))
@@ -1096,7 +1091,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
                         span,
                         name,
                         expected_element: element_ty1,
-                        second_arg: arg_tys[1],
+                        second_arg: args[1].layout.ty,
                         in_elem,
                         in_ty,
                         mutability: ExpectedPointerMutability::Mut,
@@ -1194,8 +1189,8 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
                 return_error!(InvalidMonomorphization::ExpectedVectorElementType {
                     span,
                     name,
-                    expected_element: arg_tys[0].simd_size_and_type(bx.tcx()).1,
-                    vector_type: arg_tys[0],
+                    expected_element: args[0].layout.ty.simd_size_and_type(bx.tcx()).1,
+                    vector_type: args[0].layout.ty,
                 });
             }
         };
diff --git a/compiler/rustc_codegen_gcc/src/lib.rs b/compiler/rustc_codegen_gcc/src/lib.rs
index 6994c385fc8..0591ffa42e4 100644
--- a/compiler/rustc_codegen_gcc/src/lib.rs
+++ b/compiler/rustc_codegen_gcc/src/lib.rs
@@ -16,7 +16,7 @@
 #![allow(internal_features)]
 #![doc(rust_logo)]
 #![feature(rustdoc_internals)]
-#![feature(rustc_private, decl_macro, never_type, trusted_len)]
+#![feature(rustc_private)]
 #![allow(broken_intra_doc_links)]
 #![recursion_limit = "256"]
 #![warn(rust_2018_idioms)]
@@ -391,7 +391,7 @@ impl WriteBackendMethods for GccCodegenBackend {
         unimplemented!()
     }
 
-    unsafe fn optimize(
+    fn optimize(
         _cgcx: &CodegenContext<Self>,
         _dcx: DiagCtxtHandle<'_>,
         module: &mut ModuleCodegen<Self::Module>,
@@ -409,14 +409,14 @@ impl WriteBackendMethods for GccCodegenBackend {
         Ok(())
     }
 
-    unsafe fn optimize_thin(
+    fn optimize_thin(
         cgcx: &CodegenContext<Self>,
         thin: ThinModule<Self>,
     ) -> Result<ModuleCodegen<Self::Module>, FatalError> {
         back::lto::optimize_thin_module(thin, cgcx)
     }
 
-    unsafe fn codegen(
+    fn codegen(
         cgcx: &CodegenContext<Self>,
         dcx: DiagCtxtHandle<'_>,
         module: ModuleCodegen<Self::Module>,
diff --git a/compiler/rustc_codegen_gcc/src/mono_item.rs b/compiler/rustc_codegen_gcc/src/mono_item.rs
index a2df7b2596f..539e3ac8507 100644
--- a/compiler/rustc_codegen_gcc/src/mono_item.rs
+++ b/compiler/rustc_codegen_gcc/src/mono_item.rs
@@ -16,7 +16,7 @@ use crate::{attributes, base};
 impl<'gcc, 'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
     #[cfg_attr(not(feature = "master"), allow(unused_variables))]
     fn predefine_static(
-        &self,
+        &mut self,
         def_id: DefId,
         _linkage: Linkage,
         visibility: Visibility,
@@ -42,7 +42,7 @@ impl<'gcc, 'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
 
     #[cfg_attr(not(feature = "master"), allow(unused_variables))]
     fn predefine_fn(
-        &self,
+        &mut self,
         instance: Instance<'tcx>,
         linkage: Linkage,
         visibility: Visibility,
diff --git a/compiler/rustc_codegen_llvm/Cargo.toml b/compiler/rustc_codegen_llvm/Cargo.toml
index bf8ec8c3b91..88efc8ac96b 100644
--- a/compiler/rustc_codegen_llvm/Cargo.toml
+++ b/compiler/rustc_codegen_llvm/Cargo.toml
@@ -15,7 +15,7 @@ gimli = "0.31"
 itertools = "0.12"
 libc = "0.2"
 measureme = "12.0.1"
-object = { version = "0.36.3", default-features = false, features = ["std", "read"] }
+object = { version = "0.37.0", default-features = false, features = ["std", "read"] }
 rustc-demangle = "0.1.21"
 rustc_abi = { path = "../rustc_abi" }
 rustc_ast = { path = "../rustc_ast" }
diff --git a/compiler/rustc_codegen_llvm/messages.ftl b/compiler/rustc_codegen_llvm/messages.ftl
index 41391b096cc..bda121c67fb 100644
--- a/compiler/rustc_codegen_llvm/messages.ftl
+++ b/compiler/rustc_codegen_llvm/messages.ftl
@@ -19,12 +19,6 @@ codegen_llvm_from_llvm_diag = {$message}
 
 codegen_llvm_from_llvm_optimization_diag = {$filename}:{$line}:{$column} {$pass_name} ({$kind}): {$message}
 
-codegen_llvm_invalid_minimum_alignment_not_power_of_two =
-    invalid minimum global alignment: {$align} is not power of 2
-
-codegen_llvm_invalid_minimum_alignment_too_large =
-    invalid minimum global alignment: {$align} is too large
-
 codegen_llvm_load_bitcode = failed to load bitcode of module "{$name}"
 codegen_llvm_load_bitcode_with_llvm_err = failed to load bitcode of module "{$name}": {$llvm_err}
 
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs
index 8294e29d07d..119cd634f98 100644
--- a/compiler/rustc_codegen_llvm/src/abi.rs
+++ b/compiler/rustc_codegen_llvm/src/abi.rs
@@ -2,7 +2,10 @@ use std::borrow::Borrow;
 use std::cmp;
 
 use libc::c_uint;
-use rustc_abi::{BackendRepr, HasDataLayout, Primitive, Reg, RegKind, Size};
+use rustc_abi::{
+    ArmCall, BackendRepr, CanonAbi, HasDataLayout, InterruptKind, Primitive, Reg, RegKind, Size,
+    X86Call,
+};
 use rustc_codegen_ssa::MemFlags;
 use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
 use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
@@ -12,7 +15,7 @@ use rustc_middle::ty::layout::LayoutOf;
 use rustc_middle::{bug, ty};
 use rustc_session::config;
 use rustc_target::callconv::{
-    ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, Conv, FnAbi, PassMode,
+    ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, FnAbi, PassMode,
 };
 use rustc_target::spec::SanitizerSet;
 use smallvec::SmallVec;
@@ -172,7 +175,6 @@ impl LlvmType for CastTarget {
 }
 
 trait ArgAbiExt<'ll, 'tcx> {
-    fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
     fn store(
         &self,
         bx: &mut Builder<'_, 'll, 'tcx>,
@@ -188,12 +190,6 @@ trait ArgAbiExt<'ll, 'tcx> {
 }
 
 impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
-    /// Gets the LLVM type for a place of the original Rust type of
-    /// this argument/return, i.e., the result of `type_of::type_of`.
-    fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type {
-        self.layout.llvm_type(cx)
-    }
-
     /// Stores a direct/indirect value described by this ArgAbi into a
     /// place for the original Rust type of this argument/return.
     /// Can be used for both storing formal arguments into Rust variables
@@ -302,9 +298,6 @@ impl<'ll, 'tcx> ArgAbiBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
     ) {
         arg_abi.store(self, val, dst)
     }
-    fn arg_memory_ty(&self, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>) -> &'ll Type {
-        arg_abi.memory_ty(self)
-    }
 }
 
 pub(crate) trait FnAbiLlvmExt<'ll, 'tcx> {
@@ -419,11 +412,17 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         if !self.can_unwind {
             func_attrs.push(llvm::AttributeKind::NoUnwind.create_attr(cx.llcx));
         }
-        if let Conv::RiscvInterrupt { kind } = self.conv {
-            func_attrs.push(llvm::CreateAttrStringValue(cx.llcx, "interrupt", kind.as_str()));
-        }
-        if let Conv::CCmseNonSecureEntry = self.conv {
-            func_attrs.push(llvm::CreateAttrString(cx.llcx, "cmse_nonsecure_entry"))
+        match self.conv {
+            CanonAbi::Interrupt(InterruptKind::RiscvMachine) => {
+                func_attrs.push(llvm::CreateAttrStringValue(cx.llcx, "interrupt", "machine"))
+            }
+            CanonAbi::Interrupt(InterruptKind::RiscvSupervisor) => {
+                func_attrs.push(llvm::CreateAttrStringValue(cx.llcx, "interrupt", "supervisor"))
+            }
+            CanonAbi::Arm(ArmCall::CCmseNonSecureEntry) => {
+                func_attrs.push(llvm::CreateAttrString(cx.llcx, "cmse_nonsecure_entry"))
+            }
+            _ => (),
         }
         attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &{ func_attrs });
 
@@ -610,7 +609,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             llvm::SetInstructionCallConv(callsite, cconv);
         }
 
-        if self.conv == Conv::CCmseNonSecureCall {
+        if self.conv == CanonAbi::Arm(ArmCall::CCmseNonSecureCall) {
             // This will probably get ignored on all targets but those supporting the TrustZone-M
             // extension (thumbv8m targets).
             let cmse_nonsecure_call = llvm::CreateAttrString(bx.cx.llcx, "cmse_nonsecure_call");
@@ -646,17 +645,11 @@ impl AbiBuilderMethods for Builder<'_, '_, '_> {
 }
 
 impl llvm::CallConv {
-    pub(crate) fn from_conv(conv: Conv, arch: &str) -> Self {
+    pub(crate) fn from_conv(conv: CanonAbi, arch: &str) -> Self {
         match conv {
-            Conv::C
-            | Conv::Rust
-            | Conv::CCmseNonSecureCall
-            | Conv::CCmseNonSecureEntry
-            | Conv::RiscvInterrupt { .. } => llvm::CCallConv,
-            Conv::Cold => llvm::ColdCallConv,
-            Conv::PreserveMost => llvm::PreserveMost,
-            Conv::PreserveAll => llvm::PreserveAll,
-            Conv::GpuKernel => {
+            CanonAbi::C | CanonAbi::Rust => llvm::CCallConv,
+            CanonAbi::RustCold => llvm::PreserveMost,
+            CanonAbi::GpuKernel => {
                 if arch == "amdgpu" {
                     llvm::AmdgpuKernel
                 } else if arch == "nvptx64" {
@@ -665,17 +658,25 @@ impl llvm::CallConv {
                     panic!("Architecture {arch} does not support GpuKernel calling convention");
                 }
             }
-            Conv::AvrInterrupt => llvm::AvrInterrupt,
-            Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt,
-            Conv::ArmAapcs => llvm::ArmAapcsCallConv,
-            Conv::Msp430Intr => llvm::Msp430Intr,
-            Conv::X86Fastcall => llvm::X86FastcallCallConv,
-            Conv::X86Intr => llvm::X86_Intr,
-            Conv::X86Stdcall => llvm::X86StdcallCallConv,
-            Conv::X86ThisCall => llvm::X86_ThisCall,
-            Conv::X86VectorCall => llvm::X86_VectorCall,
-            Conv::X86_64SysV => llvm::X86_64_SysV,
-            Conv::X86_64Win64 => llvm::X86_64_Win64,
+            CanonAbi::Interrupt(interrupt_kind) => match interrupt_kind {
+                InterruptKind::Avr => llvm::AvrInterrupt,
+                InterruptKind::AvrNonBlocking => llvm::AvrNonBlockingInterrupt,
+                InterruptKind::Msp430 => llvm::Msp430Intr,
+                InterruptKind::RiscvMachine | InterruptKind::RiscvSupervisor => llvm::CCallConv,
+                InterruptKind::X86 => llvm::X86_Intr,
+            },
+            CanonAbi::Arm(arm_call) => match arm_call {
+                ArmCall::Aapcs => llvm::ArmAapcsCallConv,
+                ArmCall::CCmseNonSecureCall | ArmCall::CCmseNonSecureEntry => llvm::CCallConv,
+            },
+            CanonAbi::X86(x86_call) => match x86_call {
+                X86Call::Fastcall => llvm::X86FastcallCallConv,
+                X86Call::Stdcall => llvm::X86StdcallCallConv,
+                X86Call::SysV64 => llvm::X86_64_SysV,
+                X86Call::Thiscall => llvm::X86_ThisCall,
+                X86Call::Vectorcall => llvm::X86_VectorCall,
+                X86Call::Win64 => llvm::X86_64_Win64,
+            },
         }
     }
 }
diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs
index 9e3893d5314..4185aef8b31 100644
--- a/compiler/rustc_codegen_llvm/src/asm.rs
+++ b/compiler/rustc_codegen_llvm/src/asm.rs
@@ -251,7 +251,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                 InlineAsmArch::Nvptx64 => {}
                 InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {}
                 InlineAsmArch::Hexagon => {}
-                InlineAsmArch::LoongArch64 => {
+                InlineAsmArch::LoongArch32 | InlineAsmArch::LoongArch64 => {
                     constraints.extend_from_slice(&[
                         "~{$fcc0}".to_string(),
                         "~{$fcc1}".to_string(),
diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs
index cb329323f5d..ee46b49a094 100644
--- a/compiler/rustc_codegen_llvm/src/back/lto.rs
+++ b/compiler/rustc_codegen_llvm/src/back/lto.rs
@@ -799,7 +799,7 @@ impl Drop for ThinBuffer {
     }
 }
 
-pub(crate) unsafe fn optimize_thin_module(
+pub(crate) fn optimize_thin_module(
     thin_module: ThinModule<LlvmCodegenBackend>,
     cgcx: &CodegenContext<LlvmCodegenBackend>,
 ) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> {
diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs
index 20721c74608..bde6a9cf4bc 100644
--- a/compiler/rustc_codegen_llvm/src/back/write.rs
+++ b/compiler/rustc_codegen_llvm/src/back/write.rs
@@ -704,7 +704,7 @@ pub(crate) unsafe fn llvm_optimize(
 }
 
 // Unsafe due to LLVM calls.
-pub(crate) unsafe fn optimize(
+pub(crate) fn optimize(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
     dcx: DiagCtxtHandle<'_>,
     module: &mut ModuleCodegen<ModuleLlvm>,
@@ -815,7 +815,7 @@ pub(crate) fn link(
     Ok(modules.remove(0))
 }
 
-pub(crate) unsafe fn codegen(
+pub(crate) fn codegen(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
     dcx: DiagCtxtHandle<'_>,
     module: ModuleCodegen<ModuleLlvm>,
diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs
index e4fac35aa44..5dda836988c 100644
--- a/compiler/rustc_codegen_llvm/src/base.rs
+++ b/compiler/rustc_codegen_llvm/src/base.rs
@@ -86,17 +86,24 @@ pub(crate) fn compile_codegen_unit(
             let mut cx = CodegenCx::new(tcx, cgu, &llvm_module);
             let mono_items = cx.codegen_unit.items_in_deterministic_order(cx.tcx);
             for &(mono_item, data) in &mono_items {
-                mono_item.predefine::<Builder<'_, '_, '_>>(&cx, data.linkage, data.visibility);
+                mono_item.predefine::<Builder<'_, '_, '_>>(
+                    &mut cx,
+                    cgu_name.as_str(),
+                    data.linkage,
+                    data.visibility,
+                );
             }
 
             // ... and now that we have everything pre-defined, fill out those definitions.
             for &(mono_item, item_data) in &mono_items {
-                mono_item.define::<Builder<'_, '_, '_>>(&mut cx, item_data);
+                mono_item.define::<Builder<'_, '_, '_>>(&mut cx, cgu_name.as_str(), item_data);
             }
 
             // If this codegen unit contains the main function, also create the
             // wrapper here
-            if let Some(entry) = maybe_create_entry_wrapper::<Builder<'_, '_, '_>>(&cx) {
+            if let Some(entry) =
+                maybe_create_entry_wrapper::<Builder<'_, '_, '_>>(&cx, cx.codegen_unit)
+            {
                 let attrs = attributes::sanitize_attrs(&cx, SanitizerSet::empty());
                 attributes::apply_to_llfn(entry, llvm::AttributePlace::Function, &attrs);
             }
@@ -108,14 +115,11 @@ pub(crate) fn compile_codegen_unit(
             }
 
             // Create the llvm.used and llvm.compiler.used variables.
-            if !cx.used_statics.borrow().is_empty() {
-                cx.create_used_variable_impl(c"llvm.used", &*cx.used_statics.borrow());
+            if !cx.used_statics.is_empty() {
+                cx.create_used_variable_impl(c"llvm.used", &cx.used_statics);
             }
-            if !cx.compiler_used_statics.borrow().is_empty() {
-                cx.create_used_variable_impl(
-                    c"llvm.compiler.used",
-                    &*cx.compiler_used_statics.borrow(),
-                );
+            if !cx.compiler_used_statics.is_empty() {
+                cx.create_used_variable_impl(c"llvm.compiler.used", &cx.compiler_used_statics);
             }
 
             // Run replace-all-uses-with for statics that need it. This must
diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs
index 5238755c8eb..ec006b59192 100644
--- a/compiler/rustc_codegen_llvm/src/builder.rs
+++ b/compiler/rustc_codegen_llvm/src/builder.rs
@@ -612,7 +612,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         &mut self,
         ty: &'ll Type,
         ptr: &'ll Value,
-        order: rustc_codegen_ssa::common::AtomicOrdering,
+        order: rustc_middle::ty::AtomicOrdering,
         size: Size,
     ) -> &'ll Value {
         unsafe {
@@ -851,7 +851,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         &mut self,
         val: &'ll Value,
         ptr: &'ll Value,
-        order: rustc_codegen_ssa::common::AtomicOrdering,
+        order: rustc_middle::ty::AtomicOrdering,
         size: Size,
     ) {
         debug!("Store {:?} -> {:?}", val, ptr);
@@ -1307,8 +1307,8 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         dst: &'ll Value,
         cmp: &'ll Value,
         src: &'ll Value,
-        order: rustc_codegen_ssa::common::AtomicOrdering,
-        failure_order: rustc_codegen_ssa::common::AtomicOrdering,
+        order: rustc_middle::ty::AtomicOrdering,
+        failure_order: rustc_middle::ty::AtomicOrdering,
         weak: bool,
     ) -> (&'ll Value, &'ll Value) {
         let weak = if weak { llvm::True } else { llvm::False };
@@ -1334,7 +1334,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         op: rustc_codegen_ssa::common::AtomicRmwBinOp,
         dst: &'ll Value,
         mut src: &'ll Value,
-        order: rustc_codegen_ssa::common::AtomicOrdering,
+        order: rustc_middle::ty::AtomicOrdering,
     ) -> &'ll Value {
         // The only RMW operation that LLVM supports on pointers is compare-exchange.
         let requires_cast_to_int = self.val_ty(src) == self.type_ptr()
@@ -1360,7 +1360,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 
     fn atomic_fence(
         &mut self,
-        order: rustc_codegen_ssa::common::AtomicOrdering,
+        order: rustc_middle::ty::AtomicOrdering,
         scope: SynchronizationScope,
     ) {
         let single_threaded = match scope {
@@ -1452,9 +1452,15 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> {
     fn get_static(&mut self, def_id: DefId) -> &'ll Value {
         // Forward to the `get_static` method of `CodegenCx`
-        let s = self.cx().get_static(def_id);
-        // Cast to default address space if globals are in a different addrspace
-        self.cx().const_pointercast(s, self.type_ptr())
+        let global = self.cx().get_static(def_id);
+        if self.cx().tcx.is_thread_local_static(def_id) {
+            let pointer = self.call_intrinsic("llvm.threadlocal.address", &[global]);
+            // Cast to default address space if globals are in a different addrspace
+            self.pointercast(pointer, self.type_ptr())
+        } else {
+            // Cast to default address space if globals are in a different addrspace
+            self.cx().const_pointercast(global, self.type_ptr())
+        }
     }
 }
 
@@ -1809,8 +1815,11 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
             let typeid_metadata = self.cx.typeid_metadata(typeid).unwrap();
             let dbg_loc = self.get_dbg_loc();
 
-            // Test whether the function pointer is associated with the type identifier.
-            let cond = self.type_test(llfn, typeid_metadata);
+            // Test whether the function pointer is associated with the type identifier using the
+            // llvm.type.test intrinsic. The LowerTypeTests link-time optimization pass replaces
+            // calls to this intrinsic with code to test type membership.
+            let typeid = self.get_metadata_value(typeid_metadata);
+            let cond = self.call_intrinsic("llvm.type.test", &[llfn, typeid]);
             let bb_pass = self.append_sibling_block("type_test.pass");
             let bb_fail = self.append_sibling_block("type_test.fail");
             self.cond_br(cond, bb_pass, bb_fail);
diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs
index bf81eb648f8..a4492d76c3c 100644
--- a/compiler/rustc_codegen_llvm/src/consts.rs
+++ b/compiler/rustc_codegen_llvm/src/consts.rs
@@ -1,8 +1,6 @@
 use std::ops::Range;
 
-use rustc_abi::{
-    Align, AlignFromBytesError, HasDataLayout, Primitive, Scalar, Size, WrappingRange,
-};
+use rustc_abi::{Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange};
 use rustc_codegen_ssa::common;
 use rustc_codegen_ssa::traits::*;
 use rustc_hir::LangItem;
@@ -20,9 +18,7 @@ use rustc_middle::{bug, span_bug};
 use tracing::{debug, instrument, trace};
 
 use crate::common::{AsCCharPtr, CodegenCx};
-use crate::errors::{
-    InvalidMinimumAlignmentNotPowerOfTwo, InvalidMinimumAlignmentTooLarge, SymbolAlreadyDefined,
-};
+use crate::errors::SymbolAlreadyDefined;
 use crate::llvm::{self, True};
 use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
@@ -149,22 +145,10 @@ fn set_global_alignment<'ll>(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align:
     // The target may require greater alignment for globals than the type does.
     // Note: GCC and Clang also allow `__attribute__((aligned))` on variables,
     // which can force it to be smaller. Rust doesn't support this yet.
-    if let Some(min) = cx.sess().target.min_global_align {
-        match Align::from_bits(min) {
-            Ok(min) => align = align.max(min),
-            Err(err) => match err {
-                AlignFromBytesError::NotPowerOfTwo(align) => {
-                    cx.sess().dcx().emit_err(InvalidMinimumAlignmentNotPowerOfTwo { align });
-                }
-                AlignFromBytesError::TooLarge(align) => {
-                    cx.sess().dcx().emit_err(InvalidMinimumAlignmentTooLarge { align });
-                }
-            },
-        }
-    }
-    unsafe {
-        llvm::LLVMSetAlignment(gv, align.bytes() as u32);
+    if let Some(min_global) = cx.sess().target.min_global_align {
+        align = Ord::max(align, min_global);
     }
+    llvm::set_alignment(gv, align);
 }
 
 fn check_and_apply_linkage<'ll, 'tcx>(
@@ -411,7 +395,7 @@ impl<'ll> CodegenCx<'ll, '_> {
         g
     }
 
-    fn codegen_static_item(&self, def_id: DefId) {
+    fn codegen_static_item(&mut self, def_id: DefId) {
         unsafe {
             assert!(
                 llvm::LLVMGetInitializer(
@@ -527,7 +511,7 @@ impl<'ll> CodegenCx<'ll, '_> {
 
             base::set_variable_sanitizer_attrs(g, attrs);
 
-            if attrs.flags.contains(CodegenFnAttrFlags::USED) {
+            if attrs.flags.contains(CodegenFnAttrFlags::USED_COMPILER) {
                 // `USED` and `USED_LINKER` can't be used together.
                 assert!(!attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER));
 
@@ -541,22 +525,33 @@ impl<'ll> CodegenCx<'ll, '_> {
                 // in the handling of `.init_array` (the static constructor list) in versions of
                 // the gold linker (prior to the one released with binutils 2.36).
                 //
-                // That said, we only ever emit these when compiling for ELF targets, unless
-                // `#[used(compiler)]` is explicitly requested. This is to avoid similar breakage
-                // on other targets, in particular MachO targets have *their* static constructor
-                // lists broken if `llvm.compiler.used` is emitted rather than `llvm.used`. However,
-                // that check happens when assigning the `CodegenFnAttrFlags` in
-                // `rustc_hir_analysis`, so we don't need to take care of it here.
+                // That said, we only ever emit these when `#[used(compiler)]` is explicitly
+                // requested. This is to avoid similar breakage on other targets, in particular
+                // MachO targets have *their* static constructor lists broken if `llvm.compiler.used`
+                // is emitted rather than `llvm.used`. However, that check happens when assigning
+                // the `CodegenFnAttrFlags` in the `codegen_fn_attrs` query, so we don't need to
+                // take care of it here.
                 self.add_compiler_used_global(g);
             }
             if attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER) {
                 // `USED` and `USED_LINKER` can't be used together.
-                assert!(!attrs.flags.contains(CodegenFnAttrFlags::USED));
+                assert!(!attrs.flags.contains(CodegenFnAttrFlags::USED_COMPILER));
 
                 self.add_used_global(g);
             }
         }
     }
+
+    /// Add a global value to a list to be stored in the `llvm.used` variable, an array of ptr.
+    pub(crate) fn add_used_global(&mut self, global: &'ll Value) {
+        self.used_statics.push(global);
+    }
+
+    /// Add a global value to a list to be stored in the `llvm.compiler.used` variable,
+    /// an array of ptr.
+    pub(crate) fn add_compiler_used_global(&mut self, global: &'ll Value) {
+        self.compiler_used_statics.push(global);
+    }
 }
 
 impl<'ll> StaticCodegenMethods for CodegenCx<'ll, '_> {
@@ -571,18 +566,7 @@ impl<'ll> StaticCodegenMethods for CodegenCx<'ll, '_> {
         self.const_pointercast(gv, self.type_ptr())
     }
 
-    fn codegen_static(&self, def_id: DefId) {
+    fn codegen_static(&mut self, def_id: DefId) {
         self.codegen_static_item(def_id)
     }
-
-    /// Add a global value to a list to be stored in the `llvm.used` variable, an array of ptr.
-    fn add_used_global(&self, global: &'ll Value) {
-        self.used_statics.borrow_mut().push(global);
-    }
-
-    /// Add a global value to a list to be stored in the `llvm.compiler.used` variable,
-    /// an array of ptr.
-    fn add_compiler_used_global(&self, global: &'ll Value) {
-        self.compiler_used_statics.borrow_mut().push(global);
-    }
 }
diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs
index b0d8e11d1fb..8d6e1d8941b 100644
--- a/compiler/rustc_codegen_llvm/src/context.rs
+++ b/compiler/rustc_codegen_llvm/src/context.rs
@@ -2,7 +2,7 @@ use std::borrow::Borrow;
 use std::cell::{Cell, RefCell};
 use std::ffi::{CStr, c_char, c_uint};
 use std::marker::PhantomData;
-use std::ops::Deref;
+use std::ops::{Deref, DerefMut};
 use std::str;
 
 use rustc_abi::{HasDataLayout, Size, TargetDataLayout, VariantIdx};
@@ -77,6 +77,13 @@ impl<'ll, T: Borrow<SCx<'ll>>> Deref for GenericCx<'ll, T> {
     }
 }
 
+impl<'ll, T: Borrow<SCx<'ll>>> DerefMut for GenericCx<'ll, T> {
+    #[inline]
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
 pub(crate) type SimpleCx<'ll> = GenericCx<'ll, SCx<'ll>>;
 
 /// There is one `CodegenCx` per codegen unit. Each one has its own LLVM
@@ -110,11 +117,11 @@ pub(crate) struct FullCx<'ll, 'tcx> {
 
     /// Statics that will be placed in the llvm.used variable
     /// See <https://llvm.org/docs/LangRef.html#the-llvm-used-global-variable> for details
-    pub used_statics: RefCell<Vec<&'ll Value>>,
+    pub used_statics: Vec<&'ll Value>,
 
     /// Statics that will be placed in the llvm.compiler.used variable
     /// See <https://llvm.org/docs/LangRef.html#the-llvm-compiler-used-global-variable> for details
-    pub compiler_used_statics: RefCell<Vec<&'ll Value>>,
+    pub compiler_used_statics: Vec<&'ll Value>,
 
     /// Mapping of non-scalar types to llvm types.
     pub type_lowering: RefCell<FxHashMap<(Ty<'tcx>, Option<VariantIdx>), &'ll Type>>,
@@ -606,8 +613,8 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
                 const_str_cache: Default::default(),
                 const_globals: Default::default(),
                 statics_to_rauw: RefCell::new(Vec::new()),
-                used_statics: RefCell::new(Vec::new()),
-                compiler_used_statics: RefCell::new(Vec::new()),
+                used_statics: Vec::new(),
+                compiler_used_statics: Vec::new(),
                 type_lowering: Default::default(),
                 scalar_lltypes: Default::default(),
                 coverage_cx,
@@ -801,10 +808,6 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         self.tcx.sess
     }
 
-    fn codegen_unit(&self) -> &'tcx CodegenUnit<'tcx> {
-        self.codegen_unit
-    }
-
     fn set_frame_pointer_type(&self, llfn: &'ll Value) {
         if let Some(attr) = attributes::frame_pointer_type_attr(self) {
             attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[attr]);
@@ -1240,6 +1243,7 @@ impl<'ll> CodegenCx<'ll, '_> {
         }
 
         ifn!("llvm.ptrmask", fn(ptr, t_isize) -> ptr);
+        ifn!("llvm.threadlocal.address", fn(ptr) -> ptr);
 
         None
     }
diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs
index c207df2fb0b..f6000e72840 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs
@@ -155,20 +155,6 @@ pub(crate) struct Regions {
 impl Regions {
     /// Returns true if none of this structure's tables contain any regions.
     pub(crate) fn has_no_regions(&self) -> bool {
-        // Every region has a span, so if there are no spans then there are no regions.
-        self.all_cov_spans().next().is_none()
-    }
-
-    pub(crate) fn all_cov_spans(&self) -> impl Iterator<Item = &CoverageSpan> {
-        macro_rules! iter_cov_spans {
-            ( $( $regions:expr ),* $(,)? ) => {
-                std::iter::empty()
-                $(
-                    .chain( $regions.iter().map(|region| &region.cov_span) )
-                )*
-            }
-        }
-
         let Self {
             code_regions,
             expansion_regions,
@@ -177,13 +163,11 @@ impl Regions {
             mcdc_decision_regions,
         } = self;
 
-        iter_cov_spans!(
-            code_regions,
-            expansion_regions,
-            branch_regions,
-            mcdc_branch_regions,
-            mcdc_decision_regions,
-        )
+        code_regions.is_empty()
+            && expansion_regions.is_empty()
+            && branch_regions.is_empty()
+            && mcdc_branch_regions.is_empty()
+            && mcdc_decision_regions.is_empty()
     }
 }
 
diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
index 55b1e728b70..a9be833a643 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
@@ -2,9 +2,7 @@ use std::sync::Arc;
 
 use itertools::Itertools;
 use rustc_abi::Align;
-use rustc_codegen_ssa::traits::{
-    BaseTypeCodegenMethods, ConstCodegenMethods, StaticCodegenMethods,
-};
+use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, ConstCodegenMethods};
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_index::IndexVec;
 use rustc_middle::ty::TyCtxt;
@@ -27,7 +25,7 @@ mod unused;
 ///
 /// Those sections are then read and understood by LLVM's `llvm-cov` tool,
 /// which is distributed in the `llvm-tools` rustup component.
-pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) {
+pub(crate) fn finalize(cx: &mut CodegenCx<'_, '_>) {
     let tcx = cx.tcx;
 
     // Ensure that LLVM is using a version of the coverage mapping format that
@@ -62,6 +60,7 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) {
         .sorted_by_cached_key(|&instance| tcx.symbol_name(instance).name)
         .filter_map(|instance| prepare_covfun_record(tcx, instance, true))
         .collect::<Vec<_>>();
+    drop(instances_used);
 
     // In a single designated CGU, also prepare covfun records for functions
     // in this crate that were instrumented for coverage, but are unused.
@@ -206,7 +205,7 @@ impl VirtualFileMapping {
 /// Generates the contents of the covmap record for this CGU, which mostly
 /// consists of a header and a list of filenames. The record is then stored
 /// as a global variable in the `__llvm_covmap` section.
-fn generate_covmap_record<'ll>(cx: &CodegenCx<'ll, '_>, version: u32, filenames_buffer: &[u8]) {
+fn generate_covmap_record<'ll>(cx: &mut CodegenCx<'ll, '_>, version: u32, filenames_buffer: &[u8]) {
     // A covmap record consists of four target-endian u32 values, followed by
     // the encoded filenames table. Two of the header fields are unused in
     // modern versions of the LLVM coverage mapping format, and are always 0.
diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs
index d3a815fabe7..b704cf2b1cd 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs
@@ -8,10 +8,7 @@ use std::ffi::CString;
 use std::sync::Arc;
 
 use rustc_abi::Align;
-use rustc_codegen_ssa::traits::{
-    BaseTypeCodegenMethods as _, ConstCodegenMethods, StaticCodegenMethods,
-};
-use rustc_index::IndexVec;
+use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods as _, ConstCodegenMethods};
 use rustc_middle::mir::coverage::{
     BasicCoverageBlock, CovTerm, CoverageIdsInfo, Expression, FunctionCoverageInfo, Mapping,
     MappingKind, Op,
@@ -105,16 +102,6 @@ fn fill_region_tables<'tcx>(
     ids_info: &'tcx CoverageIdsInfo,
     covfun: &mut CovfunRecord<'tcx>,
 ) {
-    // If this function is unused, replace all counters with zero.
-    let counter_for_bcb = |bcb: BasicCoverageBlock| -> ffi::Counter {
-        let term = if covfun.is_used {
-            ids_info.term_for_bcb[bcb].expect("every BCB in a mapping was given a term")
-        } else {
-            CovTerm::Zero
-        };
-        ffi::Counter::from_term(term)
-    };
-
     // Currently a function's mappings must all be in the same file, so use the
     // first mapping's span to determine the file.
     let source_map = tcx.sess.source_map();
@@ -126,12 +113,6 @@ fn fill_region_tables<'tcx>(
 
     let local_file_id = covfun.virtual_file_mapping.push_file(&source_file);
 
-    // If this testing flag is set, add an extra unused entry to the local
-    // file table, to help test the code for detecting unused file IDs.
-    if tcx.sess.coverage_inject_unused_local_file() {
-        covfun.virtual_file_mapping.push_file(&source_file);
-    }
-
     // In rare cases, _all_ of a function's spans are discarded, and coverage
     // codegen needs to handle that gracefully to avoid #133606.
     // It's hard for tests to trigger this organically, so instead we set
@@ -152,6 +133,16 @@ fn fill_region_tables<'tcx>(
     // For each counter/region pair in this function+file, convert it to a
     // form suitable for FFI.
     for &Mapping { ref kind, span } in &fn_cov_info.mappings {
+        // If this function is unused, replace all counters with zero.
+        let counter_for_bcb = |bcb: BasicCoverageBlock| -> ffi::Counter {
+            let term = if covfun.is_used {
+                ids_info.term_for_bcb[bcb].expect("every BCB in a mapping was given a term")
+            } else {
+                CovTerm::Zero
+            };
+            ffi::Counter::from_term(term)
+        };
+
         let Some(coords) = make_coords(span) else { continue };
         let cov_span = coords.make_coverage_span(local_file_id);
 
@@ -184,31 +175,14 @@ fn fill_region_tables<'tcx>(
     }
 }
 
-/// LLVM requires all local file IDs to have at least one mapping region.
-/// If that's not the case, skip this function, to avoid an assertion failure
-/// (or worse) in LLVM.
-fn check_local_file_table(covfun: &CovfunRecord<'_>) -> bool {
-    let mut local_file_id_seen =
-        IndexVec::<u32, _>::from_elem_n(false, covfun.virtual_file_mapping.local_file_table.len());
-    for cov_span in covfun.regions.all_cov_spans() {
-        local_file_id_seen[cov_span.file_id] = true;
-    }
-
-    local_file_id_seen.into_iter().all(|seen| seen)
-}
-
 /// Generates the contents of the covfun record for this function, which
 /// contains the function's coverage mapping data. The record is then stored
 /// as a global variable in the `__llvm_covfun` section.
 pub(crate) fn generate_covfun_record<'tcx>(
-    cx: &CodegenCx<'_, 'tcx>,
+    cx: &mut CodegenCx<'_, 'tcx>,
     global_file_table: &GlobalFileTable,
     covfun: &CovfunRecord<'tcx>,
 ) {
-    if !check_local_file_table(covfun) {
-        return;
-    }
-
     let &CovfunRecord {
         mangled_function_name,
         source_hash,
diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs
index 574463be7ff..39a59560c9d 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs
@@ -39,10 +39,7 @@ impl Coords {
 /// or other expansions), and if it does happen then skipping a span or function is
 /// better than an ICE or `llvm-cov` failure that the user might have no way to avoid.
 pub(crate) fn make_coords(source_map: &SourceMap, file: &SourceFile, span: Span) -> Option<Coords> {
-    if span.is_empty() {
-        debug_assert!(false, "can't make coords from empty span: {span:?}");
-        return None;
-    }
+    let span = ensure_non_empty_span(source_map, span)?;
 
     let lo = span.lo();
     let hi = span.hi();
@@ -73,6 +70,29 @@ pub(crate) fn make_coords(source_map: &SourceMap, file: &SourceFile, span: Span)
     })
 }
 
+fn ensure_non_empty_span(source_map: &SourceMap, span: Span) -> Option<Span> {
+    if !span.is_empty() {
+        return Some(span);
+    }
+
+    // The span is empty, so try to enlarge it to cover an adjacent '{' or '}'.
+    source_map
+        .span_to_source(span, |src, start, end| try {
+            // Adjusting span endpoints by `BytePos(1)` is normally a bug,
+            // but in this case we have specifically checked that the character
+            // we're skipping over is one of two specific ASCII characters, so
+            // adjusting by exactly 1 byte is correct.
+            if src.as_bytes().get(end).copied() == Some(b'{') {
+                Some(span.with_hi(span.hi() + BytePos(1)))
+            } else if start > 0 && src.as_bytes()[start - 1] == b'}' {
+                Some(span.with_lo(span.lo() - BytePos(1)))
+            } else {
+                None
+            }
+        })
+        .ok()?
+}
+
 /// If `llvm-cov` sees a source region that is improperly ordered (end < start),
 /// it will immediately exit with a fatal error. To prevent that from happening,
 /// discard regions that are improperly ordered, or might be interpreted in a
diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
index ea7f581a3cb..eefbd7cf6c4 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
@@ -56,7 +56,7 @@ impl<'ll, 'tcx> CguCoverageContext<'ll, 'tcx> {
 }
 
 impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
-    pub(crate) fn coverageinfo_finalize(&self) {
+    pub(crate) fn coverageinfo_finalize(&mut self) {
         mapgen::finalize(self)
     }
 
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs
index e9574108696..a5c80895741 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs
@@ -1,7 +1,7 @@
 use std::borrow::Cow;
 
 use libc::c_uint;
-use rustc_abi::{Align, Endian, Size, TagEncoding, VariantIdx, Variants};
+use rustc_abi::{Align, Endian, FieldIdx, Size, TagEncoding, VariantIdx, Variants};
 use rustc_codegen_ssa::debuginfo::type_names::compute_debuginfo_type_name;
 use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo};
 use rustc_codegen_ssa::traits::{ConstCodegenMethods, MiscCodegenMethods};
@@ -401,7 +401,7 @@ fn build_union_fields_for_enum<'ll, 'tcx>(
     enum_type_and_layout: TyAndLayout<'tcx>,
     enum_type_di_node: &'ll DIType,
     variant_indices: impl Iterator<Item = VariantIdx> + Clone,
-    tag_field: usize,
+    tag_field: FieldIdx,
     untagged_variant_index: Option<VariantIdx>,
 ) -> SmallVec<&'ll DIType> {
     let tag_base_type = tag_base_type(cx.tcx, enum_type_and_layout);
@@ -805,7 +805,7 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>(
     variant_field_infos: &[VariantFieldInfo<'ll>],
     discr_type_di_node: &'ll DIType,
     tag_base_type: Ty<'tcx>,
-    tag_field: usize,
+    tag_field: FieldIdx,
     untagged_variant_index: Option<VariantIdx>,
     di_flags: DIFlags,
 ) -> SmallVec<&'ll DIType> {
@@ -858,7 +858,7 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>(
     }));
 
     assert_eq!(
-        cx.size_and_align_of(enum_type_and_layout.field(cx, tag_field).ty),
+        cx.size_and_align_of(enum_type_and_layout.field(cx, tag_field.as_usize()).ty),
         cx.size_and_align_of(self::tag_base_type(cx.tcx, enum_type_and_layout))
     );
 
@@ -875,7 +875,7 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>(
             Endian::Big => (8, 0),
         };
 
-        let tag_field_offset = enum_type_and_layout.fields.offset(tag_field).bytes();
+        let tag_field_offset = enum_type_and_layout.fields.offset(tag_field.as_usize()).bytes();
         let lo_offset = Size::from_bytes(tag_field_offset + lo_offset);
         let hi_offset = Size::from_bytes(tag_field_offset + hi_offset);
 
@@ -905,8 +905,8 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>(
             cx,
             enum_type_di_node,
             TAG_FIELD_NAME,
-            enum_type_and_layout.field(cx, tag_field),
-            enum_type_and_layout.fields.offset(tag_field),
+            enum_type_and_layout.field(cx, tag_field.as_usize()),
+            enum_type_and_layout.fields.offset(tag_field.as_usize()),
             di_flags,
             tag_base_type_di_node,
             None,
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs
index 20a841f2287..62d38d463ab 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs
@@ -373,7 +373,7 @@ fn build_discr_member_di_node<'ll, 'tcx>(
                 file,
                 UNKNOWN_LINE_NUMBER,
                 layout,
-                enum_or_coroutine_type_and_layout.fields.offset(tag_field),
+                enum_or_coroutine_type_and_layout.fields.offset(tag_field.as_usize()),
                 DIFlags::FlagArtificial,
                 ty,
             ))
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
index c5085927923..5ca2505cec4 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
@@ -147,6 +147,12 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) {
     }
 }
 
+impl<'ll> Builder<'_, 'll, '_> {
+    pub(crate) fn get_dbg_loc(&self) -> Option<&'ll DILocation> {
+        unsafe { llvm::LLVMGetCurrentDebugLocation2(self.llbuilder) }
+    }
+}
+
 impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> {
     // FIXME(eddyb) find a common convention for all of the debuginfo-related
     // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.).
@@ -209,10 +215,6 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> {
         }
     }
 
-    fn get_dbg_loc(&self) -> Option<&'ll DILocation> {
-        unsafe { llvm::LLVMGetCurrentDebugLocation2(self.llbuilder) }
-    }
-
     fn insert_reference_to_gdb_debug_scripts_section_global(&mut self) {
         gdb::insert_reference_to_gdb_debug_scripts_section_global(self)
     }
diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs
index ecf108f988f..eaafc680712 100644
--- a/compiler/rustc_codegen_llvm/src/errors.rs
+++ b/compiler/rustc_codegen_llvm/src/errors.rs
@@ -58,18 +58,6 @@ pub(crate) struct SymbolAlreadyDefined<'a> {
 }
 
 #[derive(Diagnostic)]
-#[diag(codegen_llvm_invalid_minimum_alignment_not_power_of_two)]
-pub(crate) struct InvalidMinimumAlignmentNotPowerOfTwo {
-    pub align: u64,
-}
-
-#[derive(Diagnostic)]
-#[diag(codegen_llvm_invalid_minimum_alignment_too_large)]
-pub(crate) struct InvalidMinimumAlignmentTooLarge {
-    pub align: u64,
-}
-
-#[derive(Diagnostic)]
 #[diag(codegen_llvm_sanitizer_memtag_requires_mte)]
 pub(crate) struct SanitizerMemtagRequiresMte;
 
diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
index 5ca57375292..10697b9a71f 100644
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -15,11 +15,10 @@ use rustc_middle::ty::{self, GenericArgsRef, Ty};
 use rustc_middle::{bug, span_bug};
 use rustc_span::{Span, Symbol, sym};
 use rustc_symbol_mangling::mangle_internal_symbol;
-use rustc_target::callconv::{FnAbi, PassMode};
 use rustc_target::spec::{HasTargetSpec, PanicStrategy};
 use tracing::debug;
 
-use crate::abi::{FnAbiLlvmExt, LlvmType};
+use crate::abi::FnAbiLlvmExt;
 use crate::builder::Builder;
 use crate::context::CodegenCx;
 use crate::llvm::{self, Metadata};
@@ -165,26 +164,14 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
     fn codegen_intrinsic_call(
         &mut self,
         instance: ty::Instance<'tcx>,
-        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
         args: &[OperandRef<'tcx, &'ll Value>],
-        llresult: &'ll Value,
+        result: PlaceRef<'tcx, &'ll Value>,
         span: Span,
     ) -> Result<(), ty::Instance<'tcx>> {
         let tcx = self.tcx;
-        let callee_ty = instance.ty(tcx, self.typing_env());
 
-        let ty::FnDef(def_id, fn_args) = *callee_ty.kind() else {
-            bug!("expected fn item type, found {}", callee_ty);
-        };
-
-        let sig = callee_ty.fn_sig(tcx);
-        let sig = tcx.normalize_erasing_late_bound_regions(self.typing_env(), sig);
-        let arg_tys = sig.inputs();
-        let ret_ty = sig.output();
-        let name = tcx.item_name(def_id);
-
-        let llret_ty = self.layout_of(ret_ty).llvm_type(self);
-        let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout);
+        let name = tcx.item_name(instance.def_id());
+        let fn_args = instance.args;
 
         let simple = get_simple_intrinsic(self, name);
         let llval = match name {
@@ -255,7 +242,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     args[0].immediate(),
                     args[1].immediate(),
                     args[2].immediate(),
-                    llresult,
+                    result,
                 );
                 return Ok(());
             }
@@ -264,26 +251,26 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                 self.call_intrinsic("llvm.va_copy", &[args[0].immediate(), args[1].immediate()])
             }
             sym::va_arg => {
-                match fn_abi.ret.layout.backend_repr {
+                match result.layout.backend_repr {
                     BackendRepr::Scalar(scalar) => {
                         match scalar.primitive() {
                             Primitive::Int(..) => {
-                                if self.cx().size_of(ret_ty).bytes() < 4 {
+                                if self.cx().size_of(result.layout.ty).bytes() < 4 {
                                     // `va_arg` should not be called on an integer type
                                     // less than 4 bytes in length. If it is, promote
                                     // the integer to an `i32` and truncate the result
                                     // back to the smaller type.
                                     let promoted_result = emit_va_arg(self, args[0], tcx.types.i32);
-                                    self.trunc(promoted_result, llret_ty)
+                                    self.trunc(promoted_result, result.layout.llvm_type(self))
                                 } else {
-                                    emit_va_arg(self, args[0], ret_ty)
+                                    emit_va_arg(self, args[0], result.layout.ty)
                                 }
                             }
                             Primitive::Float(Float::F16) => {
                                 bug!("the va_arg intrinsic does not work with `f16`")
                             }
                             Primitive::Float(Float::F64) | Primitive::Pointer(_) => {
-                                emit_va_arg(self, args[0], ret_ty)
+                                emit_va_arg(self, args[0], result.layout.ty)
                             }
                             // `va_arg` should never be used with the return type f32.
                             Primitive::Float(Float::F32) => {
@@ -299,18 +286,12 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
             }
 
             sym::volatile_load | sym::unaligned_volatile_load => {
-                let tp_ty = fn_args.type_at(0);
                 let ptr = args[0].immediate();
-                let load = if let PassMode::Cast { cast: ty, pad_i32: _ } = &fn_abi.ret.mode {
-                    let llty = ty.llvm_type(self);
-                    self.volatile_load(llty, ptr)
-                } else {
-                    self.volatile_load(self.layout_of(tp_ty).llvm_type(self), ptr)
-                };
+                let load = self.volatile_load(result.layout.llvm_type(self), ptr);
                 let align = if name == sym::unaligned_volatile_load {
                     1
                 } else {
-                    self.align_of(tp_ty).bytes() as u32
+                    result.layout.align.abi.bytes() as u32
                 };
                 unsafe {
                     llvm::LLVMSetAlignment(load, align);
@@ -393,7 +374,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
             | sym::rotate_right
             | sym::saturating_add
             | sym::saturating_sub => {
-                let ty = arg_tys[0];
+                let ty = args[0].layout.ty;
                 if !ty.is_integral() {
                     tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
                         span,
@@ -412,26 +393,26 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                             &[args[0].immediate(), y],
                         );
 
-                        self.intcast(ret, llret_ty, false)
+                        self.intcast(ret, result.layout.llvm_type(self), false)
                     }
                     sym::ctlz_nonzero => {
                         let y = self.const_bool(true);
                         let llvm_name = &format!("llvm.ctlz.i{width}");
                         let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]);
-                        self.intcast(ret, llret_ty, false)
+                        self.intcast(ret, result.layout.llvm_type(self), false)
                     }
                     sym::cttz_nonzero => {
                         let y = self.const_bool(true);
                         let llvm_name = &format!("llvm.cttz.i{width}");
                         let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]);
-                        self.intcast(ret, llret_ty, false)
+                        self.intcast(ret, result.layout.llvm_type(self), false)
                     }
                     sym::ctpop => {
                         let ret = self.call_intrinsic(
                             &format!("llvm.ctpop.i{width}"),
                             &[args[0].immediate()],
                         );
-                        self.intcast(ret, llret_ty, false)
+                        self.intcast(ret, result.layout.llvm_type(self), false)
                     }
                     sym::bswap => {
                         if width == 8 {
@@ -563,16 +544,16 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                 // Unpack non-power-of-2 #[repr(packed, simd)] arguments.
                 // This gives them the expected layout of a regular #[repr(simd)] vector.
                 let mut loaded_args = Vec::new();
-                for (ty, arg) in arg_tys.iter().zip(args) {
+                for arg in args {
                     loaded_args.push(
                         // #[repr(packed, simd)] vectors are passed like arrays (as references,
                         // with reduced alignment and no padding) rather than as immediates.
                         // We can use a vector load to fix the layout and turn the argument
                         // into an immediate.
-                        if ty.is_simd()
+                        if arg.layout.ty.is_simd()
                             && let OperandValue::Ref(place) = arg.val
                         {
-                            let (size, elem_ty) = ty.simd_size_and_type(self.tcx());
+                            let (size, elem_ty) = arg.layout.ty.simd_size_and_type(self.tcx());
                             let elem_ll_ty = match elem_ty.kind() {
                                 ty::Float(f) => self.type_float_from_ty(*f),
                                 ty::Int(i) => self.type_int_from_ty(*i),
@@ -589,10 +570,10 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     );
                 }
 
-                let llret_ty = if ret_ty.is_simd()
-                    && let BackendRepr::Memory { .. } = self.layout_of(ret_ty).layout.backend_repr
+                let llret_ty = if result.layout.ty.is_simd()
+                    && let BackendRepr::Memory { .. } = result.layout.backend_repr
                 {
-                    let (size, elem_ty) = ret_ty.simd_size_and_type(self.tcx());
+                    let (size, elem_ty) = result.layout.ty.simd_size_and_type(self.tcx());
                     let elem_ll_ty = match elem_ty.kind() {
                         ty::Float(f) => self.type_float_from_ty(*f),
                         ty::Int(i) => self.type_int_from_ty(*i),
@@ -602,16 +583,15 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     };
                     self.type_vector(elem_ll_ty, size)
                 } else {
-                    llret_ty
+                    result.layout.llvm_type(self)
                 };
 
                 match generic_simd_intrinsic(
                     self,
                     name,
-                    callee_ty,
                     fn_args,
                     &loaded_args,
-                    ret_ty,
+                    result.layout.ty,
                     llret_ty,
                     span,
                 ) {
@@ -629,14 +609,11 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
             }
         };
 
-        if !fn_abi.ret.is_ignore() {
-            if let PassMode::Cast { .. } = &fn_abi.ret.mode {
-                self.store(llval, result.val.llval, result.val.align);
-            } else {
-                OperandRef::from_immediate_or_packed_pair(self, llval, result.layout)
-                    .val
-                    .store(self, result);
-            }
+        if result.layout.ty.is_bool() {
+            let val = self.from_immediate(llval);
+            self.store_to_place(val, result.val);
+        } else if !result.layout.ty.is_unit() {
+            self.store_to_place(llval, result.val);
         }
         Ok(())
     }
@@ -659,13 +636,6 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
         }
     }
 
-    fn type_test(&mut self, pointer: Self::Value, typeid: Self::Metadata) -> Self::Value {
-        // Test the called operand using llvm.type.test intrinsic. The LowerTypeTests link-time
-        // optimization pass replaces calls to this intrinsic with code to test type membership.
-        let typeid = self.get_metadata_value(typeid);
-        self.call_intrinsic("llvm.type.test", &[pointer, typeid])
-    }
-
     fn type_checked_load(
         &mut self,
         llvtable: &'ll Value,
@@ -688,20 +658,19 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
     }
 }
 
-fn catch_unwind_intrinsic<'ll>(
-    bx: &mut Builder<'_, 'll, '_>,
+fn catch_unwind_intrinsic<'ll, 'tcx>(
+    bx: &mut Builder<'_, 'll, 'tcx>,
     try_func: &'ll Value,
     data: &'ll Value,
     catch_func: &'ll Value,
-    dest: &'ll Value,
+    dest: PlaceRef<'tcx, &'ll Value>,
 ) {
     if bx.sess().panic_strategy() == PanicStrategy::Abort {
         let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void());
         bx.call(try_func_ty, None, None, try_func, &[data], None, None);
         // Return 0 unconditionally from the intrinsic call;
         // we can never unwind.
-        let ret_align = bx.tcx().data_layout.i32_align.abi;
-        bx.store(bx.const_i32(0), dest, ret_align);
+        OperandValue::Immediate(bx.const_i32(0)).store(bx, dest);
     } else if wants_msvc_seh(bx.sess()) {
         codegen_msvc_try(bx, try_func, data, catch_func, dest);
     } else if wants_wasm_eh(bx.sess()) {
@@ -720,12 +689,12 @@ fn catch_unwind_intrinsic<'ll>(
 // instructions are meant to work for all targets, as of the time of this
 // writing, however, LLVM does not recommend the usage of these new instructions
 // as the old ones are still more optimized.
-fn codegen_msvc_try<'ll>(
-    bx: &mut Builder<'_, 'll, '_>,
+fn codegen_msvc_try<'ll, 'tcx>(
+    bx: &mut Builder<'_, 'll, 'tcx>,
     try_func: &'ll Value,
     data: &'ll Value,
     catch_func: &'ll Value,
-    dest: &'ll Value,
+    dest: PlaceRef<'tcx, &'ll Value>,
 ) {
     let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
         bx.set_personality_fn(bx.eh_personality());
@@ -865,17 +834,16 @@ fn codegen_msvc_try<'ll>(
     // Note that no invoke is used here because by definition this function
     // can't panic (that's what it's catching).
     let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
-    let i32_align = bx.tcx().data_layout.i32_align.abi;
-    bx.store(ret, dest, i32_align);
+    OperandValue::Immediate(ret).store(bx, dest);
 }
 
 // WASM's definition of the `rust_try` function.
-fn codegen_wasm_try<'ll>(
-    bx: &mut Builder<'_, 'll, '_>,
+fn codegen_wasm_try<'ll, 'tcx>(
+    bx: &mut Builder<'_, 'll, 'tcx>,
     try_func: &'ll Value,
     data: &'ll Value,
     catch_func: &'ll Value,
-    dest: &'ll Value,
+    dest: PlaceRef<'tcx, &'ll Value>,
 ) {
     let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
         bx.set_personality_fn(bx.eh_personality());
@@ -939,8 +907,7 @@ fn codegen_wasm_try<'ll>(
     // Note that no invoke is used here because by definition this function
     // can't panic (that's what it's catching).
     let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
-    let i32_align = bx.tcx().data_layout.i32_align.abi;
-    bx.store(ret, dest, i32_align);
+    OperandValue::Immediate(ret).store(bx, dest);
 }
 
 // Definition of the standard `try` function for Rust using the GNU-like model
@@ -954,12 +921,12 @@ fn codegen_wasm_try<'ll>(
 // function calling it, and that function may already have other personality
 // functions in play. By calling a shim we're guaranteed that our shim will have
 // the right personality function.
-fn codegen_gnu_try<'ll>(
-    bx: &mut Builder<'_, 'll, '_>,
+fn codegen_gnu_try<'ll, 'tcx>(
+    bx: &mut Builder<'_, 'll, 'tcx>,
     try_func: &'ll Value,
     data: &'ll Value,
     catch_func: &'ll Value,
-    dest: &'ll Value,
+    dest: PlaceRef<'tcx, &'ll Value>,
 ) {
     let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
         // Codegens the shims described above:
@@ -1006,19 +973,18 @@ fn codegen_gnu_try<'ll>(
     // Note that no invoke is used here because by definition this function
     // can't panic (that's what it's catching).
     let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
-    let i32_align = bx.tcx().data_layout.i32_align.abi;
-    bx.store(ret, dest, i32_align);
+    OperandValue::Immediate(ret).store(bx, dest);
 }
 
 // Variant of codegen_gnu_try used for emscripten where Rust panics are
 // implemented using C++ exceptions. Here we use exceptions of a specific type
 // (`struct rust_panic`) to represent Rust panics.
-fn codegen_emcc_try<'ll>(
-    bx: &mut Builder<'_, 'll, '_>,
+fn codegen_emcc_try<'ll, 'tcx>(
+    bx: &mut Builder<'_, 'll, 'tcx>,
     try_func: &'ll Value,
     data: &'ll Value,
     catch_func: &'ll Value,
-    dest: &'ll Value,
+    dest: PlaceRef<'tcx, &'ll Value>,
 ) {
     let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| {
         // Codegens the shims described above:
@@ -1089,8 +1055,7 @@ fn codegen_emcc_try<'ll>(
     // Note that no invoke is used here because by definition this function
     // can't panic (that's what it's catching).
     let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None, None);
-    let i32_align = bx.tcx().data_layout.i32_align.abi;
-    bx.store(ret, dest, i32_align);
+    OperandValue::Immediate(ret).store(bx, dest);
 }
 
 // Helper function to give a Block to a closure to codegen a shim function.
@@ -1167,7 +1132,6 @@ fn get_rust_try_fn<'a, 'll, 'tcx>(
 fn generic_simd_intrinsic<'ll, 'tcx>(
     bx: &mut Builder<'_, 'll, 'tcx>,
     name: Symbol,
-    callee_ty: Ty<'tcx>,
     fn_args: GenericArgsRef<'tcx>,
     args: &[OperandRef<'tcx, &'ll Value>],
     ret_ty: Ty<'tcx>,
@@ -1238,26 +1202,22 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len))
     }
 
-    let tcx = bx.tcx();
-    let sig = tcx.normalize_erasing_late_bound_regions(bx.typing_env(), callee_ty.fn_sig(tcx));
-    let arg_tys = sig.inputs();
-
     // Sanity-check: all vector arguments must be immediates.
     if cfg!(debug_assertions) {
-        for (ty, arg) in arg_tys.iter().zip(args) {
-            if ty.is_simd() {
+        for arg in args {
+            if arg.layout.ty.is_simd() {
                 assert_matches!(arg.val, OperandValue::Immediate(_));
             }
         }
     }
 
     if name == sym::simd_select_bitmask {
-        let (len, _) = require_simd!(arg_tys[1], SimdArgument);
+        let (len, _) = require_simd!(args[1].layout.ty, SimdArgument);
 
         let expected_int_bits = len.max(8).next_power_of_two();
         let expected_bytes = len.div_ceil(8);
 
-        let mask_ty = arg_tys[0];
+        let mask_ty = args[0].layout.ty;
         let mask = match mask_ty.kind() {
             ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
             ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(),
@@ -1291,8 +1251,8 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     }
 
     // every intrinsic below takes a SIMD vector as its first argument
-    let (in_len, in_elem) = require_simd!(arg_tys[0], SimdInput);
-    let in_ty = arg_tys[0];
+    let (in_len, in_elem) = require_simd!(args[0].layout.ty, SimdInput);
+    let in_ty = args[0].layout.ty;
 
     let comparison = match name {
         sym::simd_eq => Some(BinOp::Eq),
@@ -1423,13 +1383,13 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
 
     if name == sym::simd_insert || name == sym::simd_insert_dyn {
         require!(
-            in_elem == arg_tys[2],
+            in_elem == args[2].layout.ty,
             InvalidMonomorphization::InsertedType {
                 span,
                 name,
                 in_elem,
                 in_ty,
-                out_ty: arg_tys[2]
+                out_ty: args[2].layout.ty
             }
         );
 
@@ -1480,7 +1440,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     if name == sym::simd_select {
         let m_elem_ty = in_elem;
         let m_len = in_len;
-        let (v_len, _) = require_simd!(arg_tys[1], SimdArgument);
+        let (v_len, _) = require_simd!(args[1].layout.ty, SimdArgument);
         require!(
             m_len == v_len,
             InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
@@ -1681,9 +1641,9 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         // The second argument must be a simd vector with an element type that's a pointer
         // to the element type of the first argument
         let (_, element_ty0) = require_simd!(in_ty, SimdFirst);
-        let (out_len, element_ty1) = require_simd!(arg_tys[1], SimdSecond);
+        let (out_len, element_ty1) = require_simd!(args[1].layout.ty, SimdSecond);
         // The element type of the third argument must be a signed integer type of any width:
-        let (out_len2, element_ty2) = require_simd!(arg_tys[2], SimdThird);
+        let (out_len2, element_ty2) = require_simd!(args[2].layout.ty, SimdThird);
         require_simd!(ret_ty, SimdReturn);
 
         // Of the same length:
@@ -1694,7 +1654,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
                 name,
                 in_len,
                 in_ty,
-                arg_ty: arg_tys[1],
+                arg_ty: args[1].layout.ty,
                 out_len
             }
         );
@@ -1705,7 +1665,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
                 name,
                 in_len,
                 in_ty,
-                arg_ty: arg_tys[2],
+                arg_ty: args[2].layout.ty,
                 out_len: out_len2
             }
         );
@@ -1725,7 +1685,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
                 span,
                 name,
                 expected_element: element_ty1,
-                second_arg: arg_tys[1],
+                second_arg: args[1].layout.ty,
                 in_elem,
                 in_ty,
                 mutability: ExpectedPointerMutability::Not,
@@ -1786,10 +1746,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         let (mask_len, mask_elem) = (in_len, in_elem);
 
         // The second argument must be a pointer matching the element type
-        let pointer_ty = arg_tys[1];
+        let pointer_ty = args[1].layout.ty;
 
         // The last argument is a passthrough vector providing values for disabled lanes
-        let values_ty = arg_tys[2];
+        let values_ty = args[2].layout.ty;
         let (values_len, values_elem) = require_simd!(values_ty, SimdThird);
 
         require_simd!(ret_ty, SimdReturn);
@@ -1877,10 +1837,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         let (mask_len, mask_elem) = (in_len, in_elem);
 
         // The second argument must be a pointer matching the element type
-        let pointer_ty = arg_tys[1];
+        let pointer_ty = args[1].layout.ty;
 
         // The last argument specifies the values to store to memory
-        let values_ty = arg_tys[2];
+        let values_ty = args[2].layout.ty;
         let (values_len, values_elem) = require_simd!(values_ty, SimdThird);
 
         // Of the same length:
@@ -1960,8 +1920,8 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         // The second argument must be a simd vector with an element type that's a pointer
         // to the element type of the first argument
         let (_, element_ty0) = require_simd!(in_ty, SimdFirst);
-        let (element_len1, element_ty1) = require_simd!(arg_tys[1], SimdSecond);
-        let (element_len2, element_ty2) = require_simd!(arg_tys[2], SimdThird);
+        let (element_len1, element_ty1) = require_simd!(args[1].layout.ty, SimdSecond);
+        let (element_len2, element_ty2) = require_simd!(args[2].layout.ty, SimdThird);
 
         // Of the same length:
         require!(
@@ -1971,7 +1931,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
                 name,
                 in_len,
                 in_ty,
-                arg_ty: arg_tys[1],
+                arg_ty: args[1].layout.ty,
                 out_len: element_len1
             }
         );
@@ -1982,7 +1942,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
                 name,
                 in_len,
                 in_ty,
-                arg_ty: arg_tys[2],
+                arg_ty: args[2].layout.ty,
                 out_len: element_len2
             }
         );
@@ -1997,7 +1957,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
                 span,
                 name,
                 expected_element: element_ty1,
-                second_arg: arg_tys[1],
+                second_arg: args[1].layout.ty,
                 in_elem,
                 in_ty,
                 mutability: ExpectedPointerMutability::Mut,
@@ -2519,7 +2479,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         let ptrs = args[0].immediate();
         // The second argument must be a ptr-sized integer.
         // (We don't care about the signedness, this is wrapping anyway.)
-        let (_offsets_len, offsets_elem) = arg_tys[1].simd_size_and_type(bx.tcx());
+        let (_offsets_len, offsets_elem) = args[1].layout.ty.simd_size_and_type(bx.tcx());
         if !matches!(offsets_elem.kind(), ty::Int(ty::IntTy::Isize) | ty::Uint(ty::UintTy::Usize)) {
             span_bug!(
                 span,
@@ -2543,8 +2503,8 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
                 return_error!(InvalidMonomorphization::ExpectedVectorElementType {
                     span,
                     name,
-                    expected_element: arg_tys[0].simd_size_and_type(bx.tcx()).1,
-                    vector_type: arg_tys[0]
+                    expected_element: args[0].layout.ty.simd_size_and_type(bx.tcx()).1,
+                    vector_type: args[0].layout.ty
                 });
             }
         };
diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs
index 5736314b96a..6890923a594 100644
--- a/compiler/rustc_codegen_llvm/src/lib.rs
+++ b/compiler/rustc_codegen_llvm/src/lib.rs
@@ -9,7 +9,6 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(assert_matches)]
-#![feature(exact_size_is_empty)]
 #![feature(extern_types)]
 #![feature(file_buffered)]
 #![feature(if_let_guard)]
@@ -189,13 +188,13 @@ impl WriteBackendMethods for LlvmCodegenBackend {
     ) -> Result<(Vec<LtoModuleCodegen<Self>>, Vec<WorkProduct>), FatalError> {
         back::lto::run_thin(cgcx, modules, cached_modules)
     }
-    unsafe fn optimize(
+    fn optimize(
         cgcx: &CodegenContext<Self>,
         dcx: DiagCtxtHandle<'_>,
         module: &mut ModuleCodegen<Self::Module>,
         config: &ModuleConfig,
     ) -> Result<(), FatalError> {
-        unsafe { back::write::optimize(cgcx, dcx, module, config) }
+        back::write::optimize(cgcx, dcx, module, config)
     }
     fn optimize_fat(
         cgcx: &CodegenContext<Self>,
@@ -205,19 +204,19 @@ impl WriteBackendMethods for LlvmCodegenBackend {
         let dcx = dcx.handle();
         back::lto::run_pass_manager(cgcx, dcx, module, false)
     }
-    unsafe fn optimize_thin(
+    fn optimize_thin(
         cgcx: &CodegenContext<Self>,
         thin: ThinModule<Self>,
     ) -> Result<ModuleCodegen<Self::Module>, FatalError> {
-        unsafe { back::lto::optimize_thin_module(thin, cgcx) }
+        back::lto::optimize_thin_module(thin, cgcx)
     }
-    unsafe fn codegen(
+    fn codegen(
         cgcx: &CodegenContext<Self>,
         dcx: DiagCtxtHandle<'_>,
         module: ModuleCodegen<Self::Module>,
         config: &ModuleConfig,
     ) -> Result<CompiledModule, FatalError> {
-        unsafe { back::write::codegen(cgcx, dcx, module, config) }
+        back::write::codegen(cgcx, dcx, module, config)
     }
     fn prepare_thin(
         module: ModuleCodegen<Self::Module>,
diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
index 67a66e6ec79..e27fbf94f34 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
@@ -426,14 +426,14 @@ pub(crate) enum AtomicOrdering {
 }
 
 impl AtomicOrdering {
-    pub(crate) fn from_generic(ao: rustc_codegen_ssa::common::AtomicOrdering) -> Self {
-        use rustc_codegen_ssa::common::AtomicOrdering as Common;
+    pub(crate) fn from_generic(ao: rustc_middle::ty::AtomicOrdering) -> Self {
+        use rustc_middle::ty::AtomicOrdering as Common;
         match ao {
             Common::Relaxed => Self::Monotonic,
             Common::Acquire => Self::Acquire,
             Common::Release => Self::Release,
-            Common::AcquireRelease => Self::AcquireRelease,
-            Common::SequentiallyConsistent => Self::SequentiallyConsistent,
+            Common::AcqRel => Self::AcquireRelease,
+            Common::SeqCst => Self::SequentiallyConsistent,
         }
     }
 }
diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs
index 8f57f0983ab..9718c95f38a 100644
--- a/compiler/rustc_codegen_llvm/src/llvm_util.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs
@@ -282,6 +282,14 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea
         }
         // Filter out features that are not supported by the current LLVM version
         ("riscv32" | "riscv64", "zacas") if get_version().0 < 20 => None,
+        (
+            "s390x",
+            "message-security-assist-extension12"
+            | "concurrent-functions"
+            | "miscellaneous-extensions-4"
+            | "vector-enhancements-3"
+            | "vector-packed-decimal-enhancement-3",
+        ) if get_version().0 < 20 => None,
         // Enable the evex512 target feature if an avx512 target feature is enabled.
         ("x86", s) if s.starts_with("avx512") => Some(LLVMFeature::with_dependencies(
             s,
diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs
index fdf62a08065..3f38e1e191b 100644
--- a/compiler/rustc_codegen_llvm/src/mono_item.rs
+++ b/compiler/rustc_codegen_llvm/src/mono_item.rs
@@ -16,7 +16,7 @@ use crate::{base, llvm};
 
 impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
     fn predefine_static(
-        &self,
+        &mut self,
         def_id: DefId,
         linkage: Linkage,
         visibility: Visibility,
@@ -44,7 +44,7 @@ impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
     }
 
     fn predefine_fn(
-        &self,
+        &mut self,
         instance: Instance<'tcx>,
         linkage: Linkage,
         visibility: Visibility,
diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs
index c216f0f4a09..236568590be 100644
--- a/compiler/rustc_codegen_llvm/src/va_arg.rs
+++ b/compiler/rustc_codegen_llvm/src/va_arg.rs
@@ -1,7 +1,10 @@
-use rustc_abi::{Align, Endian, HasDataLayout, Size};
+use rustc_abi::{Align, BackendRepr, Endian, HasDataLayout, Primitive, Size, TyAndLayout};
+use rustc_codegen_ssa::MemFlags;
 use rustc_codegen_ssa::common::IntPredicate;
 use rustc_codegen_ssa::mir::operand::OperandRef;
-use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods};
+use rustc_codegen_ssa::traits::{
+    BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, LayoutTypeCodegenMethods,
+};
 use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
 
@@ -37,6 +40,7 @@ fn emit_direct_ptr_va_arg<'ll, 'tcx>(
     align: Align,
     slot_size: Align,
     allow_higher_align: bool,
+    force_right_adjust: bool,
 ) -> (&'ll Value, Align) {
     let va_list_ty = bx.type_ptr();
     let va_list_addr = list.immediate();
@@ -54,7 +58,10 @@ fn emit_direct_ptr_va_arg<'ll, 'tcx>(
     let next = bx.inbounds_ptradd(addr, full_direct_size);
     bx.store(next, va_list_addr, bx.tcx().data_layout.pointer_align.abi);
 
-    if size.bytes() < slot_size.bytes() && bx.tcx().sess.target.endian == Endian::Big {
+    if size.bytes() < slot_size.bytes()
+        && bx.tcx().sess.target.endian == Endian::Big
+        && force_right_adjust
+    {
         let adjusted_size = bx.cx().const_i32((slot_size.bytes() - size.bytes()) as i32);
         let adjusted = bx.inbounds_ptradd(addr, adjusted_size);
         (adjusted, addr_align)
@@ -63,14 +70,40 @@ fn emit_direct_ptr_va_arg<'ll, 'tcx>(
     }
 }
 
+enum PassMode {
+    Direct,
+    Indirect,
+}
+
+enum SlotSize {
+    Bytes8 = 8,
+    Bytes4 = 4,
+}
+
+enum AllowHigherAlign {
+    No,
+    Yes,
+}
+
+enum ForceRightAdjust {
+    No,
+    Yes,
+}
+
 fn emit_ptr_va_arg<'ll, 'tcx>(
     bx: &mut Builder<'_, 'll, 'tcx>,
     list: OperandRef<'tcx, &'ll Value>,
     target_ty: Ty<'tcx>,
-    indirect: bool,
-    slot_size: Align,
-    allow_higher_align: bool,
+    pass_mode: PassMode,
+    slot_size: SlotSize,
+    allow_higher_align: AllowHigherAlign,
+    force_right_adjust: ForceRightAdjust,
 ) -> &'ll Value {
+    let indirect = matches!(pass_mode, PassMode::Indirect);
+    let allow_higher_align = matches!(allow_higher_align, AllowHigherAlign::Yes);
+    let force_right_adjust = matches!(force_right_adjust, ForceRightAdjust::Yes);
+    let slot_size = Align::from_bytes(slot_size as u64).unwrap();
+
     let layout = bx.cx.layout_of(target_ty);
     let (llty, size, align) = if indirect {
         (
@@ -81,8 +114,15 @@ fn emit_ptr_va_arg<'ll, 'tcx>(
     } else {
         (layout.llvm_type(bx.cx), layout.size, layout.align)
     };
-    let (addr, addr_align) =
-        emit_direct_ptr_va_arg(bx, list, size, align.abi, slot_size, allow_higher_align);
+    let (addr, addr_align) = emit_direct_ptr_va_arg(
+        bx,
+        list,
+        size,
+        align.abi,
+        slot_size,
+        allow_higher_align,
+        force_right_adjust,
+    );
     if indirect {
         let tmp_ret = bx.load(llty, addr, addr_align);
         bx.load(bx.cx.layout_of(target_ty).llvm_type(bx.cx), tmp_ret, align.abi)
@@ -179,8 +219,15 @@ fn emit_aapcs_va_arg<'ll, 'tcx>(
 
     // On Stack block
     bx.switch_to_block(on_stack);
-    let stack_value =
-        emit_ptr_va_arg(bx, list, target_ty, false, Align::from_bytes(8).unwrap(), true);
+    let stack_value = emit_ptr_va_arg(
+        bx,
+        list,
+        target_ty,
+        PassMode::Direct,
+        SlotSize::Bytes8,
+        AllowHigherAlign::Yes,
+        ForceRightAdjust::No,
+    );
     bx.br(end);
 
     bx.switch_to_block(end);
@@ -190,6 +237,150 @@ fn emit_aapcs_va_arg<'ll, 'tcx>(
     val
 }
 
+fn emit_powerpc_va_arg<'ll, 'tcx>(
+    bx: &mut Builder<'_, 'll, 'tcx>,
+    list: OperandRef<'tcx, &'ll Value>,
+    target_ty: Ty<'tcx>,
+) -> &'ll Value {
+    let dl = bx.cx.data_layout();
+
+    // struct __va_list_tag {
+    //   unsigned char gpr;
+    //   unsigned char fpr;
+    //   unsigned short reserved;
+    //   void *overflow_arg_area;
+    //   void *reg_save_area;
+    // };
+    let va_list_addr = list.immediate();
+
+    // Peel off any newtype wrappers.
+    let layout = {
+        let mut layout = bx.cx.layout_of(target_ty);
+
+        while let Some((_, inner)) = layout.non_1zst_field(bx.cx) {
+            layout = inner;
+        }
+
+        layout
+    };
+
+    // Rust does not currently support any powerpc softfloat targets.
+    let target = &bx.cx.tcx.sess.target;
+    let is_soft_float_abi = target.abi == "softfloat";
+    assert!(!is_soft_float_abi);
+
+    // All instances of VaArgSafe are passed directly.
+    let is_indirect = false;
+
+    let (is_i64, is_int, is_f64) = match layout.layout.backend_repr() {
+        BackendRepr::Scalar(scalar) => match scalar.primitive() {
+            rustc_abi::Primitive::Int(integer, _) => (integer.size().bits() == 64, true, false),
+            rustc_abi::Primitive::Float(float) => (false, false, float.size().bits() == 64),
+            rustc_abi::Primitive::Pointer(_) => (false, true, false),
+        },
+        _ => unreachable!("all instances of VaArgSafe are represented as scalars"),
+    };
+
+    let num_regs_addr = if is_int || is_soft_float_abi {
+        va_list_addr // gpr
+    } else {
+        bx.inbounds_ptradd(va_list_addr, bx.const_usize(1)) // fpr
+    };
+
+    let mut num_regs = bx.load(bx.type_i8(), num_regs_addr, dl.i8_align.abi);
+
+    // "Align" the register count when the type is passed as `i64`.
+    if is_i64 || (is_f64 && is_soft_float_abi) {
+        num_regs = bx.add(num_regs, bx.const_u8(1));
+        num_regs = bx.and(num_regs, bx.const_u8(0b1111_1110));
+    }
+
+    let max_regs = 8u8;
+    let use_regs = bx.icmp(IntPredicate::IntULT, num_regs, bx.const_u8(max_regs));
+
+    let in_reg = bx.append_sibling_block("va_arg.in_reg");
+    let in_mem = bx.append_sibling_block("va_arg.in_mem");
+    let end = bx.append_sibling_block("va_arg.end");
+
+    bx.cond_br(use_regs, in_reg, in_mem);
+
+    let reg_addr = {
+        bx.switch_to_block(in_reg);
+
+        let reg_safe_area_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(1 + 1 + 2 + 4));
+        let mut reg_addr = bx.load(bx.type_ptr(), reg_safe_area_ptr, dl.pointer_align.abi);
+
+        // Floating-point registers start after the general-purpose registers.
+        if !is_int && !is_soft_float_abi {
+            reg_addr = bx.inbounds_ptradd(reg_addr, bx.cx.const_usize(32))
+        }
+
+        // Get the address of the saved value by scaling the number of
+        // registers we've used by the number of.
+        let reg_size = if is_int || is_soft_float_abi { 4 } else { 8 };
+        let reg_offset = bx.mul(num_regs, bx.cx().const_u8(reg_size));
+        let reg_addr = bx.inbounds_ptradd(reg_addr, reg_offset);
+
+        // Increase the used-register count.
+        let reg_incr = if is_i64 || (is_f64 && is_soft_float_abi) { 2 } else { 1 };
+        let new_num_regs = bx.add(num_regs, bx.cx.const_u8(reg_incr));
+        bx.store(new_num_regs, num_regs_addr, dl.i8_align.abi);
+
+        bx.br(end);
+
+        reg_addr
+    };
+
+    let mem_addr = {
+        bx.switch_to_block(in_mem);
+
+        bx.store(bx.const_u8(max_regs), num_regs_addr, dl.i8_align.abi);
+
+        // Everything in the overflow area is rounded up to a size of at least 4.
+        let overflow_area_align = Align::from_bytes(4).unwrap();
+
+        let size = if !is_indirect {
+            layout.layout.size.align_to(overflow_area_align)
+        } else {
+            dl.pointer_size
+        };
+
+        let overflow_area_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(1 + 1 + 2));
+        let mut overflow_area = bx.load(bx.type_ptr(), overflow_area_ptr, dl.pointer_align.abi);
+
+        // Round up address of argument to alignment
+        if layout.layout.align.abi > overflow_area_align {
+            overflow_area = round_pointer_up_to_alignment(
+                bx,
+                overflow_area,
+                layout.layout.align.abi,
+                bx.type_ptr(),
+            );
+        }
+
+        let mem_addr = overflow_area;
+
+        // Increase the overflow area.
+        overflow_area = bx.inbounds_ptradd(overflow_area, bx.const_usize(size.bytes()));
+        bx.store(overflow_area, overflow_area_ptr, dl.pointer_align.abi);
+
+        bx.br(end);
+
+        mem_addr
+    };
+
+    // Return the appropriate result.
+    bx.switch_to_block(end);
+    let val_addr = bx.phi(bx.type_ptr(), &[reg_addr, mem_addr], &[in_reg, in_mem]);
+    let val_type = layout.llvm_type(bx);
+    let val_addr = if is_indirect {
+        bx.load(bx.cx.type_ptr(), val_addr, dl.pointer_align.abi)
+    } else {
+        val_addr
+    };
+    bx.load(val_type, val_addr, layout.align.abi)
+}
+
 fn emit_s390x_va_arg<'ll, 'tcx>(
     bx: &mut Builder<'_, 'll, 'tcx>,
     list: OperandRef<'tcx, &'ll Value>,
@@ -278,6 +469,313 @@ fn emit_s390x_va_arg<'ll, 'tcx>(
     bx.load(val_type, val_addr, layout.align.abi)
 }
 
+fn emit_x86_64_sysv64_va_arg<'ll, 'tcx>(
+    bx: &mut Builder<'_, 'll, 'tcx>,
+    list: OperandRef<'tcx, &'ll Value>,
+    target_ty: Ty<'tcx>,
+) -> &'ll Value {
+    let dl = bx.cx.data_layout();
+
+    // Implementation of the systemv x86_64 ABI calling convention for va_args, see
+    // https://gitlab.com/x86-psABIs/x86-64-ABI (section 3.5.7). This implementation is heavily
+    // based on the one in clang.
+
+    // We're able to take some shortcuts because the return type of `va_arg` must implement the
+    // `VaArgSafe` trait. Currently, only pointers, f64, i32, u32, i64 and u64 implement this trait.
+
+    // typedef struct __va_list_tag {
+    //     unsigned int gp_offset;
+    //     unsigned int fp_offset;
+    //     void *overflow_arg_area;
+    //     void *reg_save_area;
+    // } va_list[1];
+    let va_list_addr = list.immediate();
+
+    // Peel off any newtype wrappers.
+    //
+    // The "C" ABI does not unwrap newtypes (see `ReprOptions::inhibit_newtype_abi_optimization`).
+    // Here, we do actually want the unwrapped representation, because that is how LLVM/Clang
+    // pass such types to variadic functions.
+    //
+    // An example of a type that must be unwrapped is `Foo` below. Without the unwrapping, it has
+    // `BackendRepr::Memory`, but we need it to be `BackendRepr::Scalar` to generate correct code.
+    //
+    // ```
+    // #[repr(C)]
+    // struct Empty;
+    //
+    // #[repr(C)]
+    // struct Foo([Empty; 8], i32);
+    // ```
+    let layout = {
+        let mut layout = bx.cx.layout_of(target_ty);
+
+        while let Some((_, inner)) = layout.non_1zst_field(bx.cx) {
+            layout = inner;
+        }
+
+        layout
+    };
+
+    // AMD64-ABI 3.5.7p5: Step 1. Determine whether type may be passed
+    // in the registers. If not go to step 7.
+
+    // AMD64-ABI 3.5.7p5: Step 2. Compute num_gp to hold the number of
+    // general purpose registers needed to pass type and num_fp to hold
+    // the number of floating point registers needed.
+
+    let mut num_gp_registers = 0;
+    let mut num_fp_registers = 0;
+
+    let mut registers_for_primitive = |p| match p {
+        Primitive::Int(integer, _is_signed) => {
+            num_gp_registers += integer.size().bytes().div_ceil(8) as u32;
+        }
+        Primitive::Float(float) => {
+            num_fp_registers += float.size().bytes().div_ceil(16) as u32;
+        }
+        Primitive::Pointer(_) => {
+            num_gp_registers += 1;
+        }
+    };
+
+    match layout.layout.backend_repr() {
+        BackendRepr::Scalar(scalar) => {
+            registers_for_primitive(scalar.primitive());
+        }
+        BackendRepr::ScalarPair(scalar1, scalar2) => {
+            registers_for_primitive(scalar1.primitive());
+            registers_for_primitive(scalar2.primitive());
+        }
+        BackendRepr::SimdVector { .. } => {
+            // Because no instance of VaArgSafe uses a non-scalar `BackendRepr`.
+            unreachable!(
+                "No x86-64 SysV va_arg implementation for {:?}",
+                layout.layout.backend_repr()
+            )
+        }
+        BackendRepr::Memory { .. } => {
+            let mem_addr = x86_64_sysv64_va_arg_from_memory(bx, va_list_addr, layout);
+            return bx.load(layout.llvm_type(bx), mem_addr, layout.align.abi);
+        }
+    };
+
+    // AMD64-ABI 3.5.7p5: Step 3. Verify whether arguments fit into
+    // registers. In the case: l->gp_offset > 48 - num_gp * 8 or
+    // l->fp_offset > 176 - num_fp * 16 go to step 7.
+
+    let unsigned_int_offset = 4;
+    let ptr_offset = 8;
+    let gp_offset_ptr = va_list_addr;
+    let fp_offset_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(unsigned_int_offset));
+
+    let gp_offset_v = bx.load(bx.type_i32(), gp_offset_ptr, Align::from_bytes(8).unwrap());
+    let fp_offset_v = bx.load(bx.type_i32(), fp_offset_ptr, Align::from_bytes(4).unwrap());
+
+    let mut use_regs = bx.const_bool(false);
+
+    if num_gp_registers > 0 {
+        let max_offset_val = 48u32 - num_gp_registers * 8;
+        let fits_in_gp = bx.icmp(IntPredicate::IntULE, gp_offset_v, bx.const_u32(max_offset_val));
+        use_regs = fits_in_gp;
+    }
+
+    if num_fp_registers > 0 {
+        let max_offset_val = 176u32 - num_fp_registers * 16;
+        let fits_in_fp = bx.icmp(IntPredicate::IntULE, fp_offset_v, bx.const_u32(max_offset_val));
+        use_regs = if num_gp_registers > 0 { bx.and(use_regs, fits_in_fp) } else { fits_in_fp };
+    }
+
+    let in_reg = bx.append_sibling_block("va_arg.in_reg");
+    let in_mem = bx.append_sibling_block("va_arg.in_mem");
+    let end = bx.append_sibling_block("va_arg.end");
+
+    bx.cond_br(use_regs, in_reg, in_mem);
+
+    // Emit code to load the value if it was passed in a register.
+    bx.switch_to_block(in_reg);
+
+    // AMD64-ABI 3.5.7p5: Step 4. Fetch type from l->reg_save_area with
+    // an offset of l->gp_offset and/or l->fp_offset. This may require
+    // copying to a temporary location in case the parameter is passed
+    // in different register classes or requires an alignment greater
+    // than 8 for general purpose registers and 16 for XMM registers.
+    //
+    // FIXME(llvm): This really results in shameful code when we end up needing to
+    // collect arguments from different places; often what should result in a
+    // simple assembling of a structure from scattered addresses has many more
+    // loads than necessary. Can we clean this up?
+    let reg_save_area_ptr =
+        bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(2 * unsigned_int_offset + ptr_offset));
+    let reg_save_area_v = bx.load(bx.type_ptr(), reg_save_area_ptr, dl.pointer_align.abi);
+
+    let reg_addr = match layout.layout.backend_repr() {
+        BackendRepr::Scalar(scalar) => match scalar.primitive() {
+            Primitive::Int(_, _) | Primitive::Pointer(_) => {
+                let reg_addr = bx.inbounds_ptradd(reg_save_area_v, gp_offset_v);
+
+                // Copy into a temporary if the type is more aligned than the register save area.
+                let gp_align = Align::from_bytes(8).unwrap();
+                copy_to_temporary_if_more_aligned(bx, reg_addr, layout, gp_align)
+            }
+            Primitive::Float(_) => bx.inbounds_ptradd(reg_save_area_v, fp_offset_v),
+        },
+        BackendRepr::ScalarPair(scalar1, scalar2) => {
+            let ty_lo = bx.cx().scalar_pair_element_backend_type(layout, 0, false);
+            let ty_hi = bx.cx().scalar_pair_element_backend_type(layout, 1, false);
+
+            let align_lo = layout.field(bx.cx, 0).layout.align().abi;
+            let align_hi = layout.field(bx.cx, 1).layout.align().abi;
+
+            match (scalar1.primitive(), scalar2.primitive()) {
+                (Primitive::Float(_), Primitive::Float(_)) => {
+                    // SSE registers are spaced 16 bytes apart in the register save
+                    // area, we need to collect the two eightbytes together.
+                    // The ABI isn't explicit about this, but it seems reasonable
+                    // to assume that the slots are 16-byte aligned, since the stack is
+                    // naturally 16-byte aligned and the prologue is expected to store
+                    // all the SSE registers to the RSA.
+                    let reg_lo_addr = bx.inbounds_ptradd(reg_save_area_v, fp_offset_v);
+                    let reg_hi_addr = bx.inbounds_ptradd(reg_lo_addr, bx.const_i32(16));
+
+                    let align = layout.layout.align().abi;
+                    let tmp = bx.alloca(layout.layout.size(), align);
+
+                    let reg_lo = bx.load(ty_lo, reg_lo_addr, align_lo);
+                    let reg_hi = bx.load(ty_hi, reg_hi_addr, align_hi);
+
+                    let offset = scalar1.size(bx.cx).align_to(align_hi).bytes();
+                    let field0 = tmp;
+                    let field1 = bx.inbounds_ptradd(tmp, bx.const_u32(offset as u32));
+
+                    bx.store(reg_lo, field0, align);
+                    bx.store(reg_hi, field1, align);
+
+                    tmp
+                }
+                (Primitive::Float(_), _) | (_, Primitive::Float(_)) => {
+                    let gp_addr = bx.inbounds_ptradd(reg_save_area_v, gp_offset_v);
+                    let fp_addr = bx.inbounds_ptradd(reg_save_area_v, fp_offset_v);
+
+                    let (reg_lo_addr, reg_hi_addr) = match scalar1.primitive() {
+                        Primitive::Float(_) => (fp_addr, gp_addr),
+                        Primitive::Int(_, _) | Primitive::Pointer(_) => (gp_addr, fp_addr),
+                    };
+
+                    let tmp = bx.alloca(layout.layout.size(), layout.layout.align().abi);
+
+                    let reg_lo = bx.load(ty_lo, reg_lo_addr, align_lo);
+                    let reg_hi = bx.load(ty_hi, reg_hi_addr, align_hi);
+
+                    let offset = scalar1.size(bx.cx).align_to(align_hi).bytes();
+                    let field0 = tmp;
+                    let field1 = bx.inbounds_ptradd(tmp, bx.const_u32(offset as u32));
+
+                    bx.store(reg_lo, field0, align_lo);
+                    bx.store(reg_hi, field1, align_hi);
+
+                    tmp
+                }
+                (_, _) => {
+                    // Two integer/pointer values are just contiguous in memory.
+                    let reg_addr = bx.inbounds_ptradd(reg_save_area_v, gp_offset_v);
+
+                    // Copy into a temporary if the type is more aligned than the register save area.
+                    let gp_align = Align::from_bytes(8).unwrap();
+                    copy_to_temporary_if_more_aligned(bx, reg_addr, layout, gp_align)
+                }
+            }
+        }
+        // The Previous match on `BackendRepr` means control flow already escaped.
+        BackendRepr::SimdVector { .. } | BackendRepr::Memory { .. } => unreachable!(),
+    };
+
+    // AMD64-ABI 3.5.7p5: Step 5. Set:
+    // l->gp_offset = l->gp_offset + num_gp * 8
+    if num_gp_registers > 0 {
+        let offset = bx.const_u32(num_gp_registers * 8);
+        let sum = bx.add(gp_offset_v, offset);
+        // An alignment of 8 because `__va_list_tag` is 8-aligned and this is its first field.
+        bx.store(sum, gp_offset_ptr, Align::from_bytes(8).unwrap());
+    }
+
+    // l->fp_offset = l->fp_offset + num_fp * 16.
+    if num_fp_registers > 0 {
+        let offset = bx.const_u32(num_fp_registers * 16);
+        let sum = bx.add(fp_offset_v, offset);
+        bx.store(sum, fp_offset_ptr, Align::from_bytes(4).unwrap());
+    }
+
+    bx.br(end);
+
+    bx.switch_to_block(in_mem);
+    let mem_addr = x86_64_sysv64_va_arg_from_memory(bx, va_list_addr, layout);
+    bx.br(end);
+
+    bx.switch_to_block(end);
+
+    let val_type = layout.llvm_type(bx);
+    let val_addr = bx.phi(bx.type_ptr(), &[reg_addr, mem_addr], &[in_reg, in_mem]);
+
+    bx.load(val_type, val_addr, layout.align.abi)
+}
+
+/// Copy into a temporary if the type is more aligned than the register save area.
+fn copy_to_temporary_if_more_aligned<'ll, 'tcx>(
+    bx: &mut Builder<'_, 'll, 'tcx>,
+    reg_addr: &'ll Value,
+    layout: TyAndLayout<'tcx, Ty<'tcx>>,
+    src_align: Align,
+) -> &'ll Value {
+    if layout.layout.align.abi > src_align {
+        let tmp = bx.alloca(layout.layout.size(), layout.layout.align().abi);
+        bx.memcpy(
+            tmp,
+            layout.layout.align.abi,
+            reg_addr,
+            src_align,
+            bx.const_u32(layout.layout.size().bytes() as u32),
+            MemFlags::empty(),
+        );
+        tmp
+    } else {
+        reg_addr
+    }
+}
+
+fn x86_64_sysv64_va_arg_from_memory<'ll, 'tcx>(
+    bx: &mut Builder<'_, 'll, 'tcx>,
+    va_list_addr: &'ll Value,
+    layout: TyAndLayout<'tcx, Ty<'tcx>>,
+) -> &'ll Value {
+    let dl = bx.cx.data_layout();
+
+    let overflow_arg_area_ptr = bx.inbounds_ptradd(va_list_addr, bx.const_usize(8));
+
+    let overflow_arg_area_v = bx.load(bx.type_ptr(), overflow_arg_area_ptr, dl.pointer_align.abi);
+    // AMD64-ABI 3.5.7p5: Step 7. Align l->overflow_arg_area upwards to a 16
+    // byte boundary if alignment needed by type exceeds 8 byte boundary.
+    // It isn't stated explicitly in the standard, but in practice we use
+    // alignment greater than 16 where necessary.
+    if layout.layout.align.abi.bytes() > 8 {
+        unreachable!("all instances of VaArgSafe have an alignment <= 8");
+    }
+
+    // AMD64-ABI 3.5.7p5: Step 8. Fetch type from l->overflow_arg_area.
+    let mem_addr = overflow_arg_area_v;
+
+    // AMD64-ABI 3.5.7p5: Step 9. Set l->overflow_arg_area to:
+    // l->overflow_arg_area + sizeof(type).
+    // AMD64-ABI 3.5.7p5: Step 10. Align l->overflow_arg_area upwards to
+    // an 8 byte boundary.
+    let size_in_bytes = layout.layout.size().bytes();
+    let offset = bx.const_i32(size_in_bytes.next_multiple_of(8) as i32);
+    let overflow_arg_area = bx.inbounds_ptradd(overflow_arg_area_v, offset);
+    bx.store(overflow_arg_area, overflow_arg_area_ptr, dl.pointer_align.abi);
+
+    mem_addr
+}
+
 fn emit_xtensa_va_arg<'ll, 'tcx>(
     bx: &mut Builder<'_, 'll, 'tcx>,
     list: OperandRef<'tcx, &'ll Value>,
@@ -309,8 +807,7 @@ fn emit_xtensa_va_arg<'ll, 'tcx>(
     // (*va).va_ndx
     let va_reg_offset = 4;
     let va_ndx_offset = va_reg_offset + 4;
-    let offset_ptr =
-        bx.inbounds_gep(bx.type_i8(), va_list_addr, &[bx.cx.const_usize(va_ndx_offset)]);
+    let offset_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(va_ndx_offset));
 
     let offset = bx.load(bx.type_i32(), offset_ptr, bx.tcx().data_layout.i32_align.abi);
     let offset = round_up_to_alignment(bx, offset, layout.align.abi);
@@ -331,11 +828,10 @@ fn emit_xtensa_va_arg<'ll, 'tcx>(
     bx.store(offset_next, offset_ptr, bx.tcx().data_layout.pointer_align.abi);
 
     // (*va).va_reg
-    let regsave_area_ptr =
-        bx.inbounds_gep(bx.type_i8(), va_list_addr, &[bx.cx.const_usize(va_reg_offset)]);
+    let regsave_area_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(va_reg_offset));
     let regsave_area =
         bx.load(bx.type_ptr(), regsave_area_ptr, bx.tcx().data_layout.pointer_align.abi);
-    let regsave_value_ptr = bx.inbounds_gep(bx.type_i8(), regsave_area, &[offset]);
+    let regsave_value_ptr = bx.inbounds_ptradd(regsave_area, offset);
     bx.br(end);
 
     bx.switch_to_block(from_stack);
@@ -356,9 +852,9 @@ fn emit_xtensa_va_arg<'ll, 'tcx>(
     bx.store(offset_next_corrected, offset_ptr, bx.tcx().data_layout.pointer_align.abi);
 
     // let stack_value_ptr = unsafe { (*va).va_stk.byte_add(offset_corrected) };
-    let stack_area_ptr = bx.inbounds_gep(bx.type_i8(), va_list_addr, &[bx.cx.const_usize(0)]);
+    let stack_area_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(0));
     let stack_area = bx.load(bx.type_ptr(), stack_area_ptr, bx.tcx().data_layout.pointer_align.abi);
-    let stack_value_ptr = bx.inbounds_gep(bx.type_i8(), stack_area, &[offset_corrected]);
+    let stack_value_ptr = bx.inbounds_ptradd(stack_area, offset_corrected);
     bx.br(end);
 
     bx.switch_to_block(end);
@@ -386,30 +882,62 @@ pub(super) fn emit_va_arg<'ll, 'tcx>(
     // Determine the va_arg implementation to use. The LLVM va_arg instruction
     // is lacking in some instances, so we should only use it as a fallback.
     let target = &bx.cx.tcx.sess.target;
-    let arch = &bx.cx.tcx.sess.target.arch;
-    match &**arch {
-        // Windows x86
-        "x86" if target.is_like_windows => {
-            emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(4).unwrap(), false)
-        }
-        // Generic x86
-        "x86" => emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(4).unwrap(), true),
-        // Windows AArch64
-        "aarch64" | "arm64ec" if target.is_like_windows => {
-            emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), false)
-        }
-        // macOS / iOS AArch64
-        "aarch64" if target.is_like_darwin => {
-            emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), true)
+
+    match &*target.arch {
+        "x86" => emit_ptr_va_arg(
+            bx,
+            addr,
+            target_ty,
+            PassMode::Direct,
+            SlotSize::Bytes4,
+            if target.is_like_windows { AllowHigherAlign::No } else { AllowHigherAlign::Yes },
+            ForceRightAdjust::No,
+        ),
+        "aarch64" | "arm64ec" if target.is_like_windows || target.is_like_darwin => {
+            emit_ptr_va_arg(
+                bx,
+                addr,
+                target_ty,
+                PassMode::Direct,
+                SlotSize::Bytes8,
+                if target.is_like_windows { AllowHigherAlign::No } else { AllowHigherAlign::Yes },
+                ForceRightAdjust::No,
+            )
         }
         "aarch64" => emit_aapcs_va_arg(bx, addr, target_ty),
         "s390x" => emit_s390x_va_arg(bx, addr, target_ty),
+        "powerpc" => emit_powerpc_va_arg(bx, addr, target_ty),
+        "powerpc64" | "powerpc64le" => emit_ptr_va_arg(
+            bx,
+            addr,
+            target_ty,
+            PassMode::Direct,
+            SlotSize::Bytes8,
+            AllowHigherAlign::Yes,
+            match &*target.arch {
+                "powerpc64" => ForceRightAdjust::Yes,
+                _ => ForceRightAdjust::No,
+            },
+        ),
         // Windows x86_64
         "x86_64" if target.is_like_windows => {
             let target_ty_size = bx.cx.size_of(target_ty).bytes();
-            let indirect: bool = target_ty_size > 8 || !target_ty_size.is_power_of_two();
-            emit_ptr_va_arg(bx, addr, target_ty, indirect, Align::from_bytes(8).unwrap(), false)
+            emit_ptr_va_arg(
+                bx,
+                addr,
+                target_ty,
+                if target_ty_size > 8 || !target_ty_size.is_power_of_two() {
+                    PassMode::Indirect
+                } else {
+                    PassMode::Direct
+                },
+                SlotSize::Bytes8,
+                AllowHigherAlign::No,
+                ForceRightAdjust::No,
+            )
         }
+        // This includes `target.is_like_darwin`, which on x86_64 targets is like sysv64.
+        "x86_64" => emit_x86_64_sysv64_va_arg(bx, addr, target_ty),
         "xtensa" => emit_xtensa_va_arg(bx, addr, target_ty),
         // For all other architecture/OS combinations fall back to using
         // the LLVM va_arg instruction.
diff --git a/compiler/rustc_codegen_ssa/Cargo.toml b/compiler/rustc_codegen_ssa/Cargo.toml
index 337c6944177..e9c4c255bce 100644
--- a/compiler/rustc_codegen_ssa/Cargo.toml
+++ b/compiler/rustc_codegen_ssa/Cargo.toml
@@ -54,7 +54,7 @@ libc = "0.2.50"
 # tidy-alphabetical-end
 
 [dependencies.object]
-version = "0.36.2"
+version = "0.37.0"
 default-features = false
 features = ["read_core", "elf", "macho", "pe", "xcoff", "unaligned", "archive", "write", "wasm"]
 
diff --git a/compiler/rustc_codegen_ssa/messages.ftl b/compiler/rustc_codegen_ssa/messages.ftl
index 2621935eecf..91f6af7fb93 100644
--- a/compiler/rustc_codegen_ssa/messages.ftl
+++ b/compiler/rustc_codegen_ssa/messages.ftl
@@ -1,13 +1,13 @@
 codegen_ssa_L4Bender_exporting_symbols_unimplemented = exporting symbols not implemented yet for L4Bender
 
+codegen_ssa_aarch64_softfloat_neon = enabling the `neon` target feature on the current target is unsound due to ABI issues
+
 codegen_ssa_add_native_library = failed to add native library {$library_path}: {$error}
 
 codegen_ssa_aix_strip_not_used = using host's `strip` binary to cross-compile to AIX which is not guaranteed to work
 
 codegen_ssa_archive_build_failure = failed to build archive at `{$path}`: {$error}
 
-codegen_ssa_atomic_compare_exchange = Atomic compare-exchange intrinsic missing failure memory ordering
-
 codegen_ssa_autodiff_without_lto = using the autodiff feature requires using fat-lto
 
 codegen_ssa_bare_instruction_set = `#[instruction_set]` requires an argument
@@ -204,8 +204,6 @@ codegen_ssa_missing_cpp_build_tool_component = or a necessary component may be m
 
 codegen_ssa_missing_features = add the missing features in a `target_feature` attribute
 
-codegen_ssa_missing_memory_ordering = Atomic intrinsic missing memory ordering
-
 codegen_ssa_missing_query_depgraph =
     found CGU-reuse attribute but `-Zquery-dep-graph` was not specified
 
@@ -372,10 +370,6 @@ codegen_ssa_unexpected_parameter_name = unexpected parameter name
 codegen_ssa_unknown_archive_kind =
     Don't know how to build archive of type: {$kind}
 
-codegen_ssa_unknown_atomic_operation = unknown atomic operation
-
-codegen_ssa_unknown_atomic_ordering = unknown ordering in atomic intrinsic
-
 codegen_ssa_unknown_reuse_kind = unknown cgu-reuse-kind `{$kind}` specified
 
 codegen_ssa_unsupported_instruction_set = target does not support `#[instruction_set]`
diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs
index c5792da2678..168077260a6 100644
--- a/compiler/rustc_codegen_ssa/src/back/link.rs
+++ b/compiler/rustc_codegen_ssa/src/back/link.rs
@@ -3,10 +3,10 @@ mod raw_dylib;
 use std::collections::BTreeSet;
 use std::ffi::OsString;
 use std::fs::{File, OpenOptions, read};
-use std::io::{BufWriter, Write};
+use std::io::{BufReader, BufWriter, Write};
 use std::ops::{ControlFlow, Deref};
 use std::path::{Path, PathBuf};
-use std::process::{ExitStatus, Output, Stdio};
+use std::process::{Output, Stdio};
 use std::{env, fmt, fs, io, mem, str};
 
 use cc::windows_registry;
@@ -68,6 +68,23 @@ pub fn ensure_removed(dcx: DiagCtxtHandle<'_>, path: &Path) {
     }
 }
 
+fn check_link_info_print_request(sess: &Session, crate_types: &[CrateType]) {
+    let print_native_static_libs =
+        sess.opts.prints.iter().any(|p| p.kind == PrintKind::NativeStaticLibs);
+    let has_staticlib = crate_types.iter().any(|ct| *ct == CrateType::Staticlib);
+    if print_native_static_libs {
+        if !has_staticlib {
+            sess.dcx()
+                .warn(format!("cannot output linkage information without staticlib crate-type"));
+            sess.dcx()
+                .note(format!("consider `--crate-type staticlib` to print linkage information"));
+        } else if !sess.opts.output_types.should_link() {
+            sess.dcx()
+                .warn(format!("cannot output linkage information when --emit link is not passed"));
+        }
+    }
+}
+
 /// Performs the linkage portion of the compilation phase. This will generate all
 /// of the requested outputs for this compilation session.
 pub fn link_binary(
@@ -167,6 +184,12 @@ pub fn link_binary(
                 );
             }
 
+            if sess.target.binary_format == BinaryFormat::Elf {
+                if let Err(err) = warn_if_linked_with_gold(sess, &out_filename) {
+                    info!(?err, "Error while checking if gold was the linker");
+                }
+            }
+
             if output.is_stdout() {
                 if output.is_tty() {
                     sess.dcx().emit_err(errors::BinaryOutputToTty {
@@ -180,6 +203,8 @@ pub fn link_binary(
         }
     }
 
+    check_link_info_print_request(sess, &codegen_results.crate_info.crate_types);
+
     // Remove the temporary object file and metadata if we aren't saving temps.
     sess.time("link_binary_remove_temps", || {
         // If the user requests that temporaries are saved, don't delete any.
@@ -717,13 +742,10 @@ fn link_natively(
 
     // Invoke the system linker
     info!("{cmd:?}");
-    let retry_on_segfault = env::var("RUSTC_RETRY_LINKER_ON_SEGFAULT").is_ok();
     let unknown_arg_regex =
         Regex::new(r"(unknown|unrecognized) (command line )?(option|argument)").unwrap();
     let mut prog;
-    let mut i = 0;
     loop {
-        i += 1;
         prog = sess.time("run_linker", || exec_linker(sess, &cmd, out_filename, flavor, tmpdir));
         let Ok(ref output) = prog else {
             break;
@@ -839,54 +861,7 @@ fn link_natively(
             continue;
         }
 
-        // Here's a terribly awful hack that really shouldn't be present in any
-        // compiler. Here an environment variable is supported to automatically
-        // retry the linker invocation if the linker looks like it segfaulted.
-        //
-        // Gee that seems odd, normally segfaults are things we want to know
-        // about!  Unfortunately though in rust-lang/rust#38878 we're
-        // experiencing the linker segfaulting on Travis quite a bit which is
-        // causing quite a bit of pain to land PRs when they spuriously fail
-        // due to a segfault.
-        //
-        // The issue #38878 has some more debugging information on it as well,
-        // but this unfortunately looks like it's just a race condition in
-        // macOS's linker with some thread pool working in the background. It
-        // seems that no one currently knows a fix for this so in the meantime
-        // we're left with this...
-        if !retry_on_segfault || i > 3 {
-            break;
-        }
-        let msg_segv = "clang: error: unable to execute command: Segmentation fault: 11";
-        let msg_bus = "clang: error: unable to execute command: Bus error: 10";
-        if out.contains(msg_segv) || out.contains(msg_bus) {
-            warn!(
-                ?cmd, %out,
-                "looks like the linker segfaulted when we tried to call it, \
-                 automatically retrying again",
-            );
-            continue;
-        }
-
-        if is_illegal_instruction(&output.status) {
-            warn!(
-                ?cmd, %out, status = %output.status,
-                "looks like the linker hit an illegal instruction when we \
-                 tried to call it, automatically retrying again.",
-            );
-            continue;
-        }
-
-        #[cfg(unix)]
-        fn is_illegal_instruction(status: &ExitStatus) -> bool {
-            use std::os::unix::prelude::*;
-            status.signal() == Some(libc::SIGILL)
-        }
-
-        #[cfg(not(unix))]
-        fn is_illegal_instruction(_status: &ExitStatus) -> bool {
-            false
-        }
+        break;
     }
 
     match prog {
@@ -1986,7 +1961,7 @@ fn add_post_link_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor
 /// This method creates a synthetic object file, which contains undefined references to all symbols
 /// that are necessary for the linking. They are only present in symbol table but not actually
 /// used in any sections, so the linker will therefore pick relevant rlibs for linking, but
-/// unused `#[no_mangle]` or `#[used]` can still be discard by GC sections.
+/// unused `#[no_mangle]` or `#[used(compiler)]` can still be discard by GC sections.
 ///
 /// There's a few internal crates in the standard library (aka libcore and
 /// libstd) which actually have a circular dependence upon one another. This
@@ -2020,7 +1995,8 @@ fn add_linked_symbol_object(
 
     if file.format() == object::BinaryFormat::MachO {
         // Divide up the sections into sub-sections via symbols for dead code stripping.
-        // Without this flag, unused `#[no_mangle]` or `#[used]` cannot be discard on MachO targets.
+        // Without this flag, unused `#[no_mangle]` or `#[used(compiler)]` cannot be
+        // discard on MachO targets.
         file.set_subsections_via_symbols();
     }
 
@@ -3406,3 +3382,54 @@ fn add_lld_args(
         }
     }
 }
+
+// gold has been deprecated with binutils 2.44
+// and is known to behave incorrectly around Rust programs.
+// There have been reports of being unable to bootstrap with gold:
+// https://github.com/rust-lang/rust/issues/139425
+// Additionally, gold miscompiles SHF_GNU_RETAIN sections, which are
+// emitted with `#[used(linker)]`.
+fn warn_if_linked_with_gold(sess: &Session, path: &Path) -> Result<(), Box<dyn std::error::Error>> {
+    use object::read::elf::{FileHeader, SectionHeader};
+    use object::read::{ReadCache, ReadRef, Result};
+    use object::{Endianness, elf};
+
+    fn elf_has_gold_version_note<'a>(
+        elf: &impl FileHeader,
+        data: impl ReadRef<'a>,
+    ) -> Result<bool> {
+        let endian = elf.endian()?;
+
+        let section =
+            elf.sections(endian, data)?.section_by_name(endian, b".note.gnu.gold-version");
+        if let Some((_, section)) = section {
+            if let Some(mut notes) = section.notes(endian, data)? {
+                return Ok(notes.any(|note| {
+                    note.is_ok_and(|note| note.n_type(endian) == elf::NT_GNU_GOLD_VERSION)
+                }));
+            }
+        }
+
+        Ok(false)
+    }
+
+    let data = ReadCache::new(BufReader::new(File::open(path)?));
+
+    let was_linked_with_gold = if sess.target.pointer_width == 64 {
+        let elf = elf::FileHeader64::<Endianness>::parse(&data)?;
+        elf_has_gold_version_note(elf, &data)?
+    } else if sess.target.pointer_width == 32 {
+        let elf = elf::FileHeader32::<Endianness>::parse(&data)?;
+        elf_has_gold_version_note(elf, &data)?
+    } else {
+        return Ok(());
+    };
+
+    if was_linked_with_gold {
+        let mut warn =
+            sess.dcx().struct_warn("the gold linker is deprecated and has known bugs with Rust");
+        warn.help("consider using LLD or ld from GNU binutils instead");
+        warn.emit();
+    }
+    Ok(())
+}
diff --git a/compiler/rustc_codegen_ssa/src/back/link/raw_dylib.rs b/compiler/rustc_codegen_ssa/src/back/link/raw_dylib.rs
index 2c24378afe1..74f39022afb 100644
--- a/compiler/rustc_codegen_ssa/src/back/link/raw_dylib.rs
+++ b/compiler/rustc_codegen_ssa/src/back/link/raw_dylib.rs
@@ -287,6 +287,7 @@ fn create_elf_raw_dylib_stub(sess: &Session, soname: &str, symbols: &[DllImport]
         (Architecture::X86_64, None) => elf::EM_X86_64,
         (Architecture::X86_64_X32, None) => elf::EM_X86_64,
         (Architecture::Hexagon, None) => elf::EM_HEXAGON,
+        (Architecture::LoongArch32, None) => elf::EM_LOONGARCH,
         (Architecture::LoongArch64, None) => elf::EM_LOONGARCH,
         (Architecture::M68k, None) => elf::EM_68K,
         (Architecture::Mips, None) => elf::EM_MIPS,
diff --git a/compiler/rustc_codegen_ssa/src/back/lto.rs b/compiler/rustc_codegen_ssa/src/back/lto.rs
index 9fd984b6419..ce6fe8a191b 100644
--- a/compiler/rustc_codegen_ssa/src/back/lto.rs
+++ b/compiler/rustc_codegen_ssa/src/back/lto.rs
@@ -56,12 +56,7 @@ impl<B: WriteBackendMethods> LtoModuleCodegen<B> {
     }
 
     /// Optimize this module within the given codegen context.
-    ///
-    /// This function is unsafe as it'll return a `ModuleCodegen` still
-    /// points to LLVM data structures owned by this `LtoModuleCodegen`.
-    /// It's intended that the module returned is immediately code generated and
-    /// dropped, and then this LTO module is dropped.
-    pub unsafe fn optimize(
+    pub fn optimize(
         self,
         cgcx: &CodegenContext<B>,
     ) -> Result<ModuleCodegen<B::Module>, FatalError> {
@@ -70,7 +65,7 @@ impl<B: WriteBackendMethods> LtoModuleCodegen<B> {
                 B::optimize_fat(cgcx, &mut module)?;
                 Ok(module)
             }
-            LtoModuleCodegen::Thin(thin) => unsafe { B::optimize_thin(cgcx, thin) },
+            LtoModuleCodegen::Thin(thin) => B::optimize_thin(cgcx, thin),
         }
     }
 
@@ -85,7 +80,7 @@ impl<B: WriteBackendMethods> LtoModuleCodegen<B> {
     }
 
     /// Run autodiff on Fat LTO module
-    pub unsafe fn autodiff(
+    pub fn autodiff(
         self,
         cgcx: &CodegenContext<B>,
         diff_fncs: Vec<AutoDiffItem>,
diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs
index ec46c71b0e4..a16862c41ee 100644
--- a/compiler/rustc_codegen_ssa/src/back/metadata.rs
+++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs
@@ -348,7 +348,7 @@ pub(super) fn elf_e_flags(architecture: Architecture, sess: &Session) -> u32 {
 
             e_flags
         }
-        Architecture::LoongArch64 => {
+        Architecture::LoongArch32 | Architecture::LoongArch64 => {
             // Source: https://github.com/loongson/la-abi-specs/blob/release/laelf.adoc#e_flags-identifies-abi-type-and-version
             let mut e_flags: u32 = elf::EF_LARCH_OBJABI_V1;
 
diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
index 96aec9769d2..92b9b6e132e 100644
--- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
+++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
@@ -1,5 +1,6 @@
 use std::collections::hash_map::Entry::*;
 
+use rustc_abi::{CanonAbi, X86Call};
 use rustc_ast::expand::allocator::{ALLOCATOR_METHODS, NO_ALLOC_SHIM_IS_UNSTABLE, global_fn_name};
 use rustc_data_structures::unord::UnordMap;
 use rustc_hir::def::DefKind;
@@ -14,7 +15,6 @@ use rustc_middle::ty::{self, GenericArgKind, GenericArgsRef, Instance, SymbolNam
 use rustc_middle::util::Providers;
 use rustc_session::config::{CrateType, OomStrategy};
 use rustc_symbol_mangling::mangle_internal_symbol;
-use rustc_target::callconv::Conv;
 use rustc_target::spec::{SanitizerSet, TlsModel};
 use tracing::debug;
 
@@ -128,7 +128,7 @@ fn reachable_non_generics_provider(tcx: TyCtxt<'_>, _: LocalCrate) -> DefIdMap<S
                 } else {
                     SymbolExportKind::Text
                 },
-                used: codegen_attrs.flags.contains(CodegenFnAttrFlags::USED)
+                used: codegen_attrs.flags.contains(CodegenFnAttrFlags::USED_COMPILER)
                     || codegen_attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER)
                     || used,
             };
@@ -652,7 +652,7 @@ pub(crate) fn symbol_name_for_instance_in_crate<'tcx>(
 fn calling_convention_for_symbol<'tcx>(
     tcx: TyCtxt<'tcx>,
     symbol: ExportedSymbol<'tcx>,
-) -> (Conv, &'tcx [rustc_target::callconv::ArgAbi<'tcx, Ty<'tcx>>]) {
+) -> (CanonAbi, &'tcx [rustc_target::callconv::ArgAbi<'tcx, Ty<'tcx>>]) {
     let instance = match symbol {
         ExportedSymbol::NonGeneric(def_id) | ExportedSymbol::Generic(def_id, _)
             if tcx.is_static(def_id) =>
@@ -683,7 +683,7 @@ fn calling_convention_for_symbol<'tcx>(
         })
         .map(|fnabi| (fnabi.conv, &fnabi.args[..]))
         // FIXME(workingjubilee): why don't we know the convention here?
-        .unwrap_or((Conv::Rust, &[]))
+        .unwrap_or((CanonAbi::Rust, &[]))
 }
 
 /// This is the symbol name of the given instance as seen by the linker.
@@ -717,14 +717,14 @@ pub(crate) fn linking_symbol_name_for_instance_in_crate<'tcx>(
         _ => return undecorated,
     };
 
-    let (conv, args) = calling_convention_for_symbol(tcx, symbol);
+    let (callconv, args) = calling_convention_for_symbol(tcx, symbol);
 
     // Decorate symbols with prefixes, suffixes and total number of bytes of arguments.
     // Reference: https://docs.microsoft.com/en-us/cpp/build/reference/decorated-names?view=msvc-170
-    let (prefix, suffix) = match conv {
-        Conv::X86Fastcall => ("@", "@"),
-        Conv::X86Stdcall => ("_", "@"),
-        Conv::X86VectorCall => ("", "@@"),
+    let (prefix, suffix) = match callconv {
+        CanonAbi::X86(X86Call::Fastcall) => ("@", "@"),
+        CanonAbi::X86(X86Call::Stdcall) => ("_", "@"),
+        CanonAbi::X86(X86Call::Vectorcall) => ("", "@@"),
         _ => {
             if let Some(prefix) = prefix {
                 undecorated.insert(0, prefix);
@@ -758,9 +758,9 @@ pub(crate) fn extend_exported_symbols<'tcx>(
     symbol: ExportedSymbol<'tcx>,
     instantiating_crate: CrateNum,
 ) {
-    let (conv, _) = calling_convention_for_symbol(tcx, symbol);
+    let (callconv, _) = calling_convention_for_symbol(tcx, symbol);
 
-    if conv != Conv::GpuKernel || tcx.sess.target.os != "amdhsa" {
+    if callconv != CanonAbi::GpuKernel || tcx.sess.target.os != "amdhsa" {
         return;
     }
 
diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs
index 0fd4ed8475b..a41ca8ce28b 100644
--- a/compiler/rustc_codegen_ssa/src/back/write.rs
+++ b/compiler/rustc_codegen_ssa/src/back/write.rs
@@ -383,7 +383,7 @@ pub struct CodegenContext<B: WriteBackendMethods> {
     pub coordinator_send: Sender<Box<dyn Any + Send>>,
     /// `true` if the codegen should be run in parallel.
     ///
-    /// Depends on [`CodegenBackend::supports_parallel()`] and `-Zno_parallel_backend`.
+    /// Depends on [`ExtraBackendMethods::supports_parallel()`] and `-Zno_parallel_backend`.
     pub parallel: bool,
 }
 
@@ -416,8 +416,7 @@ fn generate_lto_work<B: ExtraBackendMethods>(
             B::run_fat_lto(cgcx, needs_fat_lto, import_only_modules).unwrap_or_else(|e| e.raise());
         if cgcx.lto == Lto::Fat && !autodiff.is_empty() {
             let config = cgcx.config(ModuleKind::Regular);
-            module =
-                unsafe { module.autodiff(cgcx, autodiff, config).unwrap_or_else(|e| e.raise()) };
+            module = module.autodiff(cgcx, autodiff, config).unwrap_or_else(|e| e.raise());
         }
         // We are adding a single work item, so the cost doesn't matter.
         vec![(WorkItem::LTO(module), 0)]
@@ -887,9 +886,7 @@ fn execute_optimize_work_item<B: ExtraBackendMethods>(
     let dcx = cgcx.create_dcx();
     let dcx = dcx.handle();
 
-    unsafe {
-        B::optimize(cgcx, dcx, &mut module, module_config)?;
-    }
+    B::optimize(cgcx, dcx, &mut module, module_config)?;
 
     // After we've done the initial round of optimizations we need to
     // decide whether to synchronously codegen this module or ship it
@@ -1020,7 +1017,7 @@ fn execute_lto_work_item<B: ExtraBackendMethods>(
     module: lto::LtoModuleCodegen<B>,
     module_config: &ModuleConfig,
 ) -> Result<WorkItemResult<B>, FatalError> {
-    let module = unsafe { module.optimize(cgcx)? };
+    let module = module.optimize(cgcx)?;
     finish_intra_module_work(cgcx, module, module_config)
 }
 
@@ -1036,7 +1033,7 @@ fn finish_intra_module_work<B: ExtraBackendMethods>(
         || module.kind == ModuleKind::Metadata
         || module.kind == ModuleKind::Allocator
     {
-        let module = unsafe { B::codegen(cgcx, dcx, module, module_config)? };
+        let module = B::codegen(cgcx, dcx, module, module_config)?;
         Ok(WorkItemResult::Finished(module))
     } else {
         Ok(WorkItemResult::NeedsLink(module))
@@ -1725,9 +1722,8 @@ fn start_executing_work<B: ExtraBackendMethods>(
             let dcx = cgcx.create_dcx();
             let dcx = dcx.handle();
             let module = B::run_link(&cgcx, dcx, needs_link).map_err(|_| ())?;
-            let module = unsafe {
-                B::codegen(&cgcx, dcx, module, cgcx.config(ModuleKind::Regular)).map_err(|_| ())?
-            };
+            let module =
+                B::codegen(&cgcx, dcx, module, cgcx.config(ModuleKind::Regular)).map_err(|_| ())?;
             compiled_modules.push(module);
         }
 
diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs
index 1890119dca7..c2d6a26de0f 100644
--- a/compiler/rustc_codegen_ssa/src/base.rs
+++ b/compiler/rustc_codegen_ssa/src/base.rs
@@ -492,6 +492,7 @@ where
 /// users main function.
 pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     cx: &'a Bx::CodegenCx,
+    cgu: &CodegenUnit<'tcx>,
 ) -> Option<Bx::Function> {
     let (main_def_id, entry_type) = cx.tcx().entry_fn(())?;
     let main_is_local = main_def_id.is_local();
@@ -500,10 +501,10 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     if main_is_local {
         // We want to create the wrapper in the same codegen unit as Rust's main
         // function.
-        if !cx.codegen_unit().contains_item(&MonoItem::Fn(instance)) {
+        if !cgu.contains_item(&MonoItem::Fn(instance)) {
             return None;
         }
-    } else if !cx.codegen_unit().is_primary() {
+    } else if !cgu.is_primary() {
         // We want to create the wrapper only when the codegen unit is the primary one
         return None;
     }
@@ -560,7 +561,7 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 
         let EntryFnType::Main { sigpipe } = entry_type;
         let (start_fn, start_ty, args, instance) = {
-            let start_def_id = cx.tcx().require_lang_item(LangItem::Start, None);
+            let start_def_id = cx.tcx().require_lang_item(LangItem::Start, DUMMY_SP);
             let start_instance = ty::Instance::expect_resolve(
                 cx.tcx(),
                 cx.typing_env(),
diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
index aa55a0e0f14..0b31fa8fa88 100644
--- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
+++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
@@ -189,41 +189,16 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
                             )
                             .emit();
                         }
-                        codegen_fn_attrs.flags |= CodegenFnAttrFlags::USED;
+                        codegen_fn_attrs.flags |= CodegenFnAttrFlags::USED_COMPILER;
                     }
                     Some(_) => {
                         tcx.dcx().emit_err(errors::ExpectedUsedSymbol { span: attr.span() });
                     }
                     None => {
-                        // Unfortunately, unconditionally using `llvm.used` causes
-                        // issues in handling `.init_array` with the gold linker,
-                        // but using `llvm.compiler.used` caused a nontrivial amount
-                        // of unintentional ecosystem breakage -- particularly on
-                        // Mach-O targets.
-                        //
-                        // As a result, we emit `llvm.compiler.used` only on ELF
-                        // targets. This is somewhat ad-hoc, but actually follows
-                        // our pre-LLVM 13 behavior (prior to the ecosystem
-                        // breakage), and seems to match `clang`'s behavior as well
-                        // (both before and after LLVM 13), possibly because they
-                        // have similar compatibility concerns to us. See
-                        // https://github.com/rust-lang/rust/issues/47384#issuecomment-1019080146
-                        // and following comments for some discussion of this, as
-                        // well as the comments in `rustc_codegen_llvm` where these
-                        // flags are handled.
-                        //
-                        // Anyway, to be clear: this is still up in the air
-                        // somewhat, and is subject to change in the future (which
-                        // is a good thing, because this would ideally be a bit
-                        // more firmed up).
-                        let is_like_elf = !(tcx.sess.target.is_like_darwin
-                            || tcx.sess.target.is_like_windows
-                            || tcx.sess.target.is_like_wasm);
-                        codegen_fn_attrs.flags |= if is_like_elf {
-                            CodegenFnAttrFlags::USED
-                        } else {
-                            CodegenFnAttrFlags::USED_LINKER
-                        };
+                        // Unconditionally using `llvm.used` causes issues in handling
+                        // `.init_array` with the gold linker. Luckily gold has been
+                        // deprecated with GCC 15 and rustc now warns about using gold.
+                        codegen_fn_attrs.flags |= CodegenFnAttrFlags::USED_LINKER
                     }
                 }
             }
@@ -299,6 +274,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
                 }
                 from_target_feature_attr(
                     tcx,
+                    did,
                     attr,
                     rust_target_features,
                     &mut codegen_fn_attrs.target_features,
diff --git a/compiler/rustc_codegen_ssa/src/common.rs b/compiler/rustc_codegen_ssa/src/common.rs
index 6d0c9d8d066..48565e0b4de 100644
--- a/compiler/rustc_codegen_ssa/src/common.rs
+++ b/compiler/rustc_codegen_ssa/src/common.rs
@@ -60,15 +60,6 @@ pub enum AtomicRmwBinOp {
 }
 
 #[derive(Copy, Clone, Debug)]
-pub enum AtomicOrdering {
-    Relaxed,
-    Acquire,
-    Release,
-    AcquireRelease,
-    SequentiallyConsistent,
-}
-
-#[derive(Copy, Clone, Debug)]
 pub enum SynchronizationScope {
     SingleThread,
     CrossThread,
@@ -119,7 +110,7 @@ mod temp_stable_hash_impls {
 
 pub(crate) fn build_langcall<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     bx: &Bx,
-    span: Option<Span>,
+    span: Span,
     li: LangItem,
 ) -> (Bx::FnAbiOfResult, Bx::Value, Instance<'tcx>) {
     let tcx = bx.tcx();
diff --git a/compiler/rustc_codegen_ssa/src/errors.rs b/compiler/rustc_codegen_ssa/src/errors.rs
index d49aac75d05..f843347db92 100644
--- a/compiler/rustc_codegen_ssa/src/errors.rs
+++ b/compiler/rustc_codegen_ssa/src/errors.rs
@@ -797,22 +797,6 @@ pub(crate) struct ShuffleIndicesEvaluation {
 }
 
 #[derive(Diagnostic)]
-#[diag(codegen_ssa_missing_memory_ordering)]
-pub(crate) struct MissingMemoryOrdering;
-
-#[derive(Diagnostic)]
-#[diag(codegen_ssa_unknown_atomic_ordering)]
-pub(crate) struct UnknownAtomicOrdering;
-
-#[derive(Diagnostic)]
-#[diag(codegen_ssa_atomic_compare_exchange)]
-pub(crate) struct AtomicCompareExchange;
-
-#[derive(Diagnostic)]
-#[diag(codegen_ssa_unknown_atomic_operation)]
-pub(crate) struct UnknownAtomicOperation;
-
-#[derive(Diagnostic)]
 pub enum InvalidMonomorphization<'tcx> {
     #[diag(codegen_ssa_invalid_monomorphization_basic_integer_type, code = E0511)]
     BasicIntegerType {
@@ -1316,3 +1300,7 @@ pub(crate) struct XcrunSdkPathWarning {
     pub sdk_name: &'static str,
     pub stderr: String,
 }
+
+#[derive(LintDiagnostic)]
+#[diag(codegen_ssa_aarch64_softfloat_neon)]
+pub(crate) struct Aarch64SoftfloatNeon;
diff --git a/compiler/rustc_codegen_ssa/src/meth.rs b/compiler/rustc_codegen_ssa/src/meth.rs
index 399c592432a..3a11ce6befb 100644
--- a/compiler/rustc_codegen_ssa/src/meth.rs
+++ b/compiler/rustc_codegen_ssa/src/meth.rs
@@ -77,7 +77,7 @@ fn dyn_trait_in_self<'tcx>(
     ty: Ty<'tcx>,
 ) -> Option<ty::ExistentialTraitRef<'tcx>> {
     for arg in ty.peel_refs().walk() {
-        if let GenericArgKind::Type(ty) = arg.unpack()
+        if let GenericArgKind::Type(ty) = arg.kind()
             && let ty::Dynamic(data, _, _) = ty.kind()
         {
             // FIXME(arbitrary_self_types): This is likely broken for receivers which
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index 950f19a6f0f..43b87171d51 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -1,6 +1,6 @@
 use std::cmp;
 
-use rustc_abi::{BackendRepr, ExternAbi, HasDataLayout, Reg, WrappingRange};
+use rustc_abi::{BackendRepr, ExternAbi, HasDataLayout, Reg, Size, WrappingRange};
 use rustc_ast as ast;
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_data_structures::packed::Pu128;
@@ -11,8 +11,8 @@ use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
 use rustc_middle::ty::{self, Instance, Ty};
 use rustc_middle::{bug, span_bug};
 use rustc_session::config::OptLevel;
+use rustc_span::Span;
 use rustc_span::source_map::Spanned;
-use rustc_span::{Span, sym};
 use rustc_target::callconv::{ArgAbi, FnAbi, PassMode};
 use tracing::{debug, info};
 
@@ -158,7 +158,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
         llargs: &[Bx::Value],
         destination: Option<(ReturnDest<'tcx, Bx::Value>, mir::BasicBlock)>,
         mut unwind: mir::UnwindAction,
-        copied_constant_arguments: &[PlaceRef<'tcx, <Bx as BackendTypes>::Value>],
+        lifetime_ends_after_call: &[(Bx::Value, Size)],
         instance: Option<Instance<'tcx>>,
         mergeable_succ: bool,
     ) -> MergingSucc {
@@ -245,8 +245,8 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
             if let Some((ret_dest, target)) = destination {
                 bx.switch_to_block(fx.llbb(target));
                 fx.set_debug_loc(bx, self.terminator.source_info);
-                for tmp in copied_constant_arguments {
-                    bx.lifetime_end(tmp.val.llval, tmp.layout.size);
+                for &(tmp, size) in lifetime_ends_after_call {
+                    bx.lifetime_end(tmp, size);
                 }
                 fx.store_return(bx, ret_dest, &fn_abi.ret, invokeret);
             }
@@ -259,8 +259,8 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
             }
 
             if let Some((ret_dest, target)) = destination {
-                for tmp in copied_constant_arguments {
-                    bx.lifetime_end(tmp.val.llval, tmp.layout.size);
+                for &(tmp, size) in lifetime_ends_after_call {
+                    bx.lifetime_end(tmp, size);
                 }
                 fx.store_return(bx, ret_dest, &fn_abi.ret, llret);
                 self.funclet_br(fx, bx, target, mergeable_succ)
@@ -783,7 +783,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             }
         };
 
-        let (fn_abi, llfn, instance) = common::build_langcall(bx, Some(span), lang_item);
+        let (fn_abi, llfn, instance) = common::build_langcall(bx, span, lang_item);
 
         // Codegen the actual panic invoke/call.
         let merging_succ =
@@ -803,7 +803,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         self.set_debug_loc(bx, terminator.source_info);
 
         // Obtain the panic entry point.
-        let (fn_abi, llfn, instance) = common::build_langcall(bx, Some(span), reason.lang_item());
+        let (fn_abi, llfn, instance) = common::build_langcall(bx, span, reason.lang_item());
 
         // Codegen the actual panic invoke/call.
         let merging_succ = helper.do_call(
@@ -827,7 +827,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         helper: &TerminatorCodegenHelper<'tcx>,
         bx: &mut Bx,
         intrinsic: ty::IntrinsicDef,
-        instance: Option<Instance<'tcx>>,
+        instance: Instance<'tcx>,
         source_info: mir::SourceInfo,
         target: Option<mir::BasicBlock>,
         unwind: mir::UnwindAction,
@@ -836,58 +836,56 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         // Emit a panic or a no-op for `assert_*` intrinsics.
         // These are intrinsics that compile to panics so that we can get a message
         // which mentions the offending type, even from a const context.
-        if let Some(requirement) = ValidityRequirement::from_intrinsic(intrinsic.name) {
-            let ty = instance.unwrap().args.type_at(0);
-
-            let do_panic = !bx
-                .tcx()
-                .check_validity_requirement((requirement, bx.typing_env().as_query_input(ty)))
-                .expect("expect to have layout during codegen");
-
-            let layout = bx.layout_of(ty);
-
-            Some(if do_panic {
-                let msg_str = with_no_visible_paths!({
-                    with_no_trimmed_paths!({
-                        if layout.is_uninhabited() {
-                            // Use this error even for the other intrinsics as it is more precise.
-                            format!("attempted to instantiate uninhabited type `{ty}`")
-                        } else if requirement == ValidityRequirement::Zero {
-                            format!("attempted to zero-initialize type `{ty}`, which is invalid")
-                        } else {
-                            format!(
-                                "attempted to leave type `{ty}` uninitialized, which is invalid"
-                            )
-                        }
-                    })
-                });
-                let msg = bx.const_str(&msg_str);
+        let Some(requirement) = ValidityRequirement::from_intrinsic(intrinsic.name) else {
+            return None;
+        };
 
-                // Obtain the panic entry point.
-                let (fn_abi, llfn, instance) =
-                    common::build_langcall(bx, Some(source_info.span), LangItem::PanicNounwind);
+        let ty = instance.args.type_at(0);
 
-                // Codegen the actual panic invoke/call.
-                helper.do_call(
-                    self,
-                    bx,
-                    fn_abi,
-                    llfn,
-                    &[msg.0, msg.1],
-                    target.as_ref().map(|bb| (ReturnDest::Nothing, *bb)),
-                    unwind,
-                    &[],
-                    Some(instance),
-                    mergeable_succ,
-                )
-            } else {
-                // a NOP
-                let target = target.unwrap();
-                helper.funclet_br(self, bx, target, mergeable_succ)
-            })
-        } else {
-            None
+        let is_valid = bx
+            .tcx()
+            .check_validity_requirement((requirement, bx.typing_env().as_query_input(ty)))
+            .expect("expect to have layout during codegen");
+
+        if is_valid {
+            // a NOP
+            let target = target.unwrap();
+            return Some(helper.funclet_br(self, bx, target, mergeable_succ));
         }
+
+        let layout = bx.layout_of(ty);
+
+        let msg_str = with_no_visible_paths!({
+            with_no_trimmed_paths!({
+                if layout.is_uninhabited() {
+                    // Use this error even for the other intrinsics as it is more precise.
+                    format!("attempted to instantiate uninhabited type `{ty}`")
+                } else if requirement == ValidityRequirement::Zero {
+                    format!("attempted to zero-initialize type `{ty}`, which is invalid")
+                } else {
+                    format!("attempted to leave type `{ty}` uninitialized, which is invalid")
+                }
+            })
+        });
+        let msg = bx.const_str(&msg_str);
+
+        // Obtain the panic entry point.
+        let (fn_abi, llfn, instance) =
+            common::build_langcall(bx, source_info.span, LangItem::PanicNounwind);
+
+        // Codegen the actual panic invoke/call.
+        Some(helper.do_call(
+            self,
+            bx,
+            fn_abi,
+            llfn,
+            &[msg.0, msg.1],
+            target.as_ref().map(|bb| (ReturnDest::Nothing, *bb)),
+            unwind,
+            &[],
+            Some(instance),
+            mergeable_succ,
+        ))
     }
 
     fn codegen_call_terminator(
@@ -903,42 +901,127 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         fn_span: Span,
         mergeable_succ: bool,
     ) -> MergingSucc {
-        let source_info = terminator.source_info;
-        let span = source_info.span;
+        let source_info = mir::SourceInfo { span: fn_span, ..terminator.source_info };
 
         // Create the callee. This is a fn ptr or zero-sized and hence a kind of scalar.
         let callee = self.codegen_operand(bx, func);
 
         let (instance, mut llfn) = match *callee.layout.ty.kind() {
-            ty::FnDef(def_id, args) => (
-                Some(ty::Instance::expect_resolve(
+            ty::FnDef(def_id, generic_args) => {
+                let instance = ty::Instance::expect_resolve(
                     bx.tcx(),
                     bx.typing_env(),
                     def_id,
-                    args,
+                    generic_args,
                     fn_span,
-                )),
-                None,
-            ),
+                );
+
+                let instance = match instance.def {
+                    // We don't need AsyncDropGlueCtorShim here because it is not `noop func`,
+                    // it is `func returning noop future`
+                    ty::InstanceKind::DropGlue(_, None) => {
+                        // Empty drop glue; a no-op.
+                        let target = target.unwrap();
+                        return helper.funclet_br(self, bx, target, mergeable_succ);
+                    }
+                    ty::InstanceKind::Intrinsic(def_id) => {
+                        let intrinsic = bx.tcx().intrinsic(def_id).unwrap();
+                        if let Some(merging_succ) = self.codegen_panic_intrinsic(
+                            &helper,
+                            bx,
+                            intrinsic,
+                            instance,
+                            source_info,
+                            target,
+                            unwind,
+                            mergeable_succ,
+                        ) {
+                            return merging_succ;
+                        }
+
+                        let result_layout =
+                            self.cx.layout_of(self.monomorphized_place_ty(destination.as_ref()));
+
+                        let (result, store_in_local) = if result_layout.is_zst() {
+                            (
+                                PlaceRef::new_sized(bx.const_undef(bx.type_ptr()), result_layout),
+                                None,
+                            )
+                        } else if let Some(local) = destination.as_local() {
+                            match self.locals[local] {
+                                LocalRef::Place(dest) => (dest, None),
+                                LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
+                                LocalRef::PendingOperand => {
+                                    // Currently, intrinsics always need a location to store
+                                    // the result, so we create a temporary `alloca` for the
+                                    // result.
+                                    let tmp = PlaceRef::alloca(bx, result_layout);
+                                    tmp.storage_live(bx);
+                                    (tmp, Some(local))
+                                }
+                                LocalRef::Operand(_) => {
+                                    bug!("place local already assigned to");
+                                }
+                            }
+                        } else {
+                            (self.codegen_place(bx, destination.as_ref()), None)
+                        };
+
+                        if result.val.align < result.layout.align.abi {
+                            // Currently, MIR code generation does not create calls
+                            // that store directly to fields of packed structs (in
+                            // fact, the calls it creates write only to temps).
+                            //
+                            // If someone changes that, please update this code path
+                            // to create a temporary.
+                            span_bug!(self.mir.span, "can't directly store to unaligned value");
+                        }
+
+                        let args: Vec<_> =
+                            args.iter().map(|arg| self.codegen_operand(bx, &arg.node)).collect();
+
+                        match self.codegen_intrinsic_call(bx, instance, &args, result, source_info)
+                        {
+                            Ok(()) => {
+                                if let Some(local) = store_in_local {
+                                    let op = bx.load_operand(result);
+                                    result.storage_dead(bx);
+                                    self.overwrite_local(local, LocalRef::Operand(op));
+                                    self.debug_introduce_local(bx, local);
+                                }
+
+                                return if let Some(target) = target {
+                                    helper.funclet_br(self, bx, target, mergeable_succ)
+                                } else {
+                                    bx.unreachable();
+                                    MergingSucc::False
+                                };
+                            }
+                            Err(instance) => {
+                                if intrinsic.must_be_overridden {
+                                    span_bug!(
+                                        fn_span,
+                                        "intrinsic {} must be overridden by codegen backend, but isn't",
+                                        intrinsic.name,
+                                    );
+                                }
+                                instance
+                            }
+                        }
+                    }
+                    _ => instance,
+                };
+
+                (Some(instance), None)
+            }
             ty::FnPtr(..) => (None, Some(callee.immediate())),
             _ => bug!("{} is not callable", callee.layout.ty),
         };
 
-        let def = instance.map(|i| i.def);
-
-        // We don't need AsyncDropGlueCtorShim here because it is not `noop func`,
-        // it is `func returning noop future`
-        if let Some(ty::InstanceKind::DropGlue(_, None)) = def {
-            // Empty drop glue; a no-op.
-            let target = target.unwrap();
-            return helper.funclet_br(self, bx, target, mergeable_succ);
-        }
-
         // FIXME(eddyb) avoid computing this if possible, when `instance` is
         // available - right now `sig` is only needed for getting the `abi`
         // and figuring out how many extra args were passed to a C-variadic `fn`.
         let sig = callee.layout.ty.fn_sig(bx.tcx());
-        let abi = sig.abi();
 
         let extra_args = &args[sig.inputs().skip_binder().len()..];
         let extra_args = bx.tcx().mk_type_list_from_iter(extra_args.iter().map(|op_arg| {
@@ -954,93 +1037,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         // The arguments we'll be passing. Plus one to account for outptr, if used.
         let arg_count = fn_abi.args.len() + fn_abi.ret.is_indirect() as usize;
 
-        let instance = match def {
-            Some(ty::InstanceKind::Intrinsic(def_id)) => {
-                let intrinsic = bx.tcx().intrinsic(def_id).unwrap();
-                if let Some(merging_succ) = self.codegen_panic_intrinsic(
-                    &helper,
-                    bx,
-                    intrinsic,
-                    instance,
-                    source_info,
-                    target,
-                    unwind,
-                    mergeable_succ,
-                ) {
-                    return merging_succ;
-                }
-
-                let mut llargs = Vec::with_capacity(1);
-                let ret_dest = self.make_return_dest(
-                    bx,
-                    destination,
-                    &fn_abi.ret,
-                    &mut llargs,
-                    Some(intrinsic),
-                );
-                let dest = match ret_dest {
-                    _ if fn_abi.ret.is_indirect() => llargs[0],
-                    ReturnDest::Nothing => bx.const_undef(bx.type_ptr()),
-                    ReturnDest::IndirectOperand(dst, _) | ReturnDest::Store(dst) => dst.val.llval,
-                    ReturnDest::DirectOperand(_) => {
-                        bug!("Cannot use direct operand with an intrinsic call")
-                    }
-                };
-
-                let args: Vec<_> =
-                    args.iter().map(|arg| self.codegen_operand(bx, &arg.node)).collect();
-
-                if matches!(intrinsic, ty::IntrinsicDef { name: sym::caller_location, .. }) {
-                    let location = self
-                        .get_caller_location(bx, mir::SourceInfo { span: fn_span, ..source_info });
-
-                    assert_eq!(llargs, []);
-                    if let ReturnDest::IndirectOperand(tmp, _) = ret_dest {
-                        location.val.store(bx, tmp);
-                    }
-                    self.store_return(bx, ret_dest, &fn_abi.ret, location.immediate());
-                    return helper.funclet_br(self, bx, target.unwrap(), mergeable_succ);
-                }
-
-                let instance = *instance.as_ref().unwrap();
-                match Self::codegen_intrinsic_call(bx, instance, fn_abi, &args, dest, span) {
-                    Ok(()) => {
-                        if let ReturnDest::IndirectOperand(dst, _) = ret_dest {
-                            self.store_return(bx, ret_dest, &fn_abi.ret, dst.val.llval);
-                        }
-
-                        return if let Some(target) = target {
-                            helper.funclet_br(self, bx, target, mergeable_succ)
-                        } else {
-                            bx.unreachable();
-                            MergingSucc::False
-                        };
-                    }
-                    Err(instance) => {
-                        if intrinsic.must_be_overridden {
-                            span_bug!(
-                                span,
-                                "intrinsic {} must be overridden by codegen backend, but isn't",
-                                intrinsic.name,
-                            );
-                        }
-                        Some(instance)
-                    }
-                }
-            }
-            _ => instance,
-        };
-
         let mut llargs = Vec::with_capacity(arg_count);
 
         // We still need to call `make_return_dest` even if there's no `target`, since
         // `fn_abi.ret` could be `PassMode::Indirect`, even if it is uninhabited,
         // and `make_return_dest` adds the return-place indirect pointer to `llargs`.
-        let return_dest = self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs, None);
+        let return_dest = self.make_return_dest(bx, destination, &fn_abi.ret, &mut llargs);
         let destination = target.map(|target| (return_dest, target));
 
         // Split the rust-call tupled arguments off.
-        let (first_args, untuple) = if abi == ExternAbi::RustCall
+        let (first_args, untuple) = if sig.abi() == ExternAbi::RustCall
             && let Some((tup, args)) = args.split_last()
         {
             (args, Some(tup))
@@ -1048,11 +1054,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             (args, None)
         };
 
-        let mut copied_constant_arguments = vec![];
+        // When generating arguments we sometimes introduce temporary allocations with lifetime
+        // that extend for the duration of a call. Keep track of those allocations and their sizes
+        // to generate `lifetime_end` when the call returns.
+        let mut lifetime_ends_after_call: Vec<(Bx::Value, Size)> = Vec::new();
         'make_args: for (i, arg) in first_args.iter().enumerate() {
             let mut op = self.codegen_operand(bx, &arg.node);
 
-            if let (0, Some(ty::InstanceKind::Virtual(_, idx))) = (i, def) {
+            if let (0, Some(ty::InstanceKind::Virtual(_, idx))) = (i, instance.map(|i| i.def)) {
                 match op.val {
                     Pair(data_ptr, meta) => {
                         // In the case of Rc<Self>, we need to explicitly pass a
@@ -1068,7 +1077,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             let (idx, _) = op.layout.non_1zst_field(bx).expect(
                                 "not exactly one non-1-ZST field in a `DispatchFromDyn` type",
                             );
-                            op = op.extract_field(self, bx, idx);
+                            op = op.extract_field(self, bx, idx.as_usize());
                         }
 
                         // Now that we have `*dyn Trait` or `&dyn Trait`, split it up into its
@@ -1100,13 +1109,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             let (idx, _) = op.layout.non_1zst_field(bx).expect(
                                 "not exactly one non-1-ZST field in a `DispatchFromDyn` type",
                             );
-                            op = op.extract_field(self, bx, idx);
+                            op = op.extract_field(self, bx, idx.as_usize());
                         }
 
                         // Make sure that we've actually unwrapped the rcvr down
                         // to a pointer or ref to `dyn* Trait`.
                         if !op.layout.ty.builtin_deref(true).unwrap().is_dyn_star() {
-                            span_bug!(span, "can't codegen a virtual call on {:#?}", op);
+                            span_bug!(fn_span, "can't codegen a virtual call on {:#?}", op);
                         }
                         let place = op.deref(bx.cx());
                         let data_place = place.project_field(bx, 0);
@@ -1122,7 +1131,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         continue;
                     }
                     _ => {
-                        span_bug!(span, "can't codegen a virtual call on {:#?}", op);
+                        span_bug!(fn_span, "can't codegen a virtual call on {:#?}", op);
                     }
                 }
             }
@@ -1136,12 +1145,18 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     bx.lifetime_start(tmp.val.llval, tmp.layout.size);
                     op.val.store(bx, tmp);
                     op.val = Ref(tmp.val);
-                    copied_constant_arguments.push(tmp);
+                    lifetime_ends_after_call.push((tmp.val.llval, tmp.layout.size));
                 }
                 _ => {}
             }
 
-            self.codegen_argument(bx, op, &mut llargs, &fn_abi.args[i]);
+            self.codegen_argument(
+                bx,
+                op,
+                &mut llargs,
+                &fn_abi.args[i],
+                &mut lifetime_ends_after_call,
+            );
         }
         let num_untupled = untuple.map(|tup| {
             self.codegen_arguments_untupled(
@@ -1149,6 +1164,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 &tup.node,
                 &mut llargs,
                 &fn_abi.args[first_args.len()..],
+                &mut lifetime_ends_after_call,
             )
         });
 
@@ -1165,22 +1181,28 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 mir_args + 1,
                 "#[track_caller] fn's must have 1 more argument in their ABI than in their MIR: {instance:?} {fn_span:?} {fn_abi:?}",
             );
-            let location =
-                self.get_caller_location(bx, mir::SourceInfo { span: fn_span, ..source_info });
+            let location = self.get_caller_location(bx, source_info);
             debug!(
                 "codegen_call_terminator({:?}): location={:?} (fn_span {:?})",
                 terminator, location, fn_span
             );
 
             let last_arg = fn_abi.args.last().unwrap();
-            self.codegen_argument(bx, location, &mut llargs, last_arg);
+            self.codegen_argument(
+                bx,
+                location,
+                &mut llargs,
+                last_arg,
+                &mut lifetime_ends_after_call,
+            );
         }
 
         let fn_ptr = match (instance, llfn) {
             (Some(instance), None) => bx.get_fn_addr(instance),
             (_, Some(llfn)) => llfn,
-            _ => span_bug!(span, "no instance or llfn for call"),
+            _ => span_bug!(fn_span, "no instance or llfn for call"),
         };
+        self.set_debug_loc(bx, source_info);
         helper.do_call(
             self,
             bx,
@@ -1189,7 +1211,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             &llargs,
             destination,
             unwind,
-            &copied_constant_arguments,
+            &lifetime_ends_after_call,
             instance,
             mergeable_succ,
         )
@@ -1479,6 +1501,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         op: OperandRef<'tcx, Bx::Value>,
         llargs: &mut Vec<Bx::Value>,
         arg: &ArgAbi<'tcx, Ty<'tcx>>,
+        lifetime_ends_after_call: &mut Vec<(Bx::Value, Size)>,
     ) {
         match arg.mode {
             PassMode::Ignore => return,
@@ -1517,7 +1540,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         None => arg.layout.align.abi,
                     };
                     let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align);
+                    bx.lifetime_start(scratch.llval, arg.layout.size);
                     op.val.store(bx, scratch.with_type(arg.layout));
+                    lifetime_ends_after_call.push((scratch.llval, arg.layout.size));
                     (scratch.llval, scratch.align, true)
                 }
                 PassMode::Cast { .. } => {
@@ -1538,7 +1563,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         // alignment requirements may be higher than the type's alignment, so copy
                         // to a higher-aligned alloca.
                         let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align);
+                        bx.lifetime_start(scratch.llval, arg.layout.size);
                         bx.typed_place_copy(scratch, op_place_val, op.layout);
+                        lifetime_ends_after_call.push((scratch.llval, arg.layout.size));
                         (scratch.llval, scratch.align, true)
                     } else {
                         (op_place_val.llval, op_place_val.align, true)
@@ -1620,6 +1647,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         operand: &mir::Operand<'tcx>,
         llargs: &mut Vec<Bx::Value>,
         args: &[ArgAbi<'tcx, Ty<'tcx>>],
+        lifetime_ends_after_call: &mut Vec<(Bx::Value, Size)>,
     ) -> usize {
         let tuple = self.codegen_operand(bx, operand);
 
@@ -1632,19 +1660,19 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             for i in 0..tuple.layout.fields.count() {
                 let field_ptr = tuple_ptr.project_field(bx, i);
                 let field = bx.load_operand(field_ptr);
-                self.codegen_argument(bx, field, llargs, &args[i]);
+                self.codegen_argument(bx, field, llargs, &args[i], lifetime_ends_after_call);
             }
         } else {
             // If the tuple is immediate, the elements are as well.
             for i in 0..tuple.layout.fields.count() {
                 let op = tuple.extract_field(self, bx, i);
-                self.codegen_argument(bx, op, llargs, &args[i]);
+                self.codegen_argument(bx, op, llargs, &args[i], lifetime_ends_after_call);
             }
         }
         tuple.layout.fields.count()
     }
 
-    fn get_caller_location(
+    pub(super) fn get_caller_location(
         &mut self,
         bx: &mut Bx,
         source_info: mir::SourceInfo,
@@ -1802,7 +1830,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
         self.set_debug_loc(&mut bx, mir::SourceInfo::outermost(self.mir.span));
 
-        let (fn_abi, fn_ptr, instance) = common::build_langcall(&bx, None, reason.lang_item());
+        let (fn_abi, fn_ptr, instance) =
+            common::build_langcall(&bx, self.mir.span, reason.lang_item());
         if is_call_from_compiler_builtins_to_upstream_monomorphization(bx.tcx(), instance) {
             bx.abort();
         } else {
@@ -1845,7 +1874,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         dest: mir::Place<'tcx>,
         fn_ret: &ArgAbi<'tcx, Ty<'tcx>>,
         llargs: &mut Vec<Bx::Value>,
-        intrinsic: Option<ty::IntrinsicDef>,
     ) -> ReturnDest<'tcx, Bx::Value> {
         // If the return is ignored, we can just return a do-nothing `ReturnDest`.
         if fn_ret.is_ignore() {
@@ -1865,13 +1893,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         tmp.storage_live(bx);
                         llargs.push(tmp.val.llval);
                         ReturnDest::IndirectOperand(tmp, index)
-                    } else if intrinsic.is_some() {
-                        // Currently, intrinsics always need a location to store
-                        // the result, so we create a temporary `alloca` for the
-                        // result.
-                        let tmp = PlaceRef::alloca(bx, fn_ret.layout);
-                        tmp.storage_live(bx);
-                        ReturnDest::IndirectOperand(tmp, index)
                     } else {
                         ReturnDest::DirectOperand(index)
                     };
@@ -1881,7 +1902,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
             }
         } else {
-            self.codegen_place(bx, mir::PlaceRef { local: dest.local, projection: dest.projection })
+            self.codegen_place(bx, dest.as_ref())
         };
         if fn_ret.is_indirect() {
             if dest.val.align < dest.layout.align.abi {
diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
index 5924c8991ad..f731613d67e 100644
--- a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
@@ -10,7 +10,7 @@ use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
 use rustc_middle::ty::{Instance, Ty};
 use rustc_middle::{bug, mir, ty};
 use rustc_session::config::DebugInfo;
-use rustc_span::{BytePos, Span, Symbol, hygiene, kw};
+use rustc_span::{BytePos, Span, Symbol, hygiene, sym};
 
 use super::operand::{OperandRef, OperandValue};
 use super::place::{PlaceRef, PlaceValue};
@@ -283,7 +283,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 // (after #67586 gets fixed).
                 None
             } else {
-                let name = kw::Empty;
+                let name = sym::empty;
                 let decl = &self.mir.local_decls[local];
                 let dbg_var = if full_debug_info {
                     self.adjusted_span_and_dbg_scope(decl.source_info).map(
@@ -318,7 +318,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             None
         } else {
             Some(match whole_local_var.or(fallback_var.clone()) {
-                Some(var) if var.name != kw::Empty => var.name.to_string(),
+                Some(var) if var.name != sym::empty => var.name.to_string(),
                 _ => format!("{local:?}"),
             })
         };
diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
index b0fcfee2adf..e217c09939e 100644
--- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
@@ -1,16 +1,17 @@
 use rustc_abi::WrappingRange;
+use rustc_middle::bug;
+use rustc_middle::mir::SourceInfo;
 use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_middle::{bug, span_bug};
 use rustc_session::config::OptLevel;
-use rustc_span::{Span, sym};
-use rustc_target::callconv::{FnAbi, PassMode};
+use rustc_span::sym;
 
 use super::FunctionCx;
 use super::operand::OperandRef;
 use super::place::PlaceRef;
+use crate::common::{AtomicRmwBinOp, SynchronizationScope};
 use crate::errors::InvalidMonomorphization;
 use crate::traits::*;
-use crate::{MemFlags, errors, meth, size_of_val};
+use crate::{MemFlags, meth, size_of_val};
 
 fn copy_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     bx: &mut Bx,
@@ -52,25 +53,17 @@ fn memset_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     /// In the `Err` case, returns the instance that should be called instead.
     pub fn codegen_intrinsic_call(
+        &mut self,
         bx: &mut Bx,
         instance: ty::Instance<'tcx>,
-        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
         args: &[OperandRef<'tcx, Bx::Value>],
-        llresult: Bx::Value,
-        span: Span,
+        result: PlaceRef<'tcx, Bx::Value>,
+        source_info: SourceInfo,
     ) -> Result<(), ty::Instance<'tcx>> {
-        let callee_ty = instance.ty(bx.tcx(), bx.typing_env());
+        let span = source_info.span;
 
-        let ty::FnDef(def_id, fn_args) = *callee_ty.kind() else {
-            span_bug!(span, "expected fn item type, found {}", callee_ty);
-        };
-
-        let sig = callee_ty.fn_sig(bx.tcx());
-        let sig = bx.tcx().normalize_erasing_late_bound_regions(bx.typing_env(), sig);
-        let arg_tys = sig.inputs();
-        let ret_ty = sig.output();
-        let name = bx.tcx().item_name(def_id);
-        let name_str = name.as_str();
+        let name = bx.tcx().item_name(instance.def_id());
+        let fn_args = instance.args;
 
         // If we're swapping something that's *not* an `OperandValue::Ref`,
         // then we can do it directly and avoid the alloca.
@@ -96,8 +89,14 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             }
         }
 
-        let llret_ty = bx.backend_type(bx.layout_of(ret_ty));
-        let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout);
+        let invalid_monomorphization_int_type = |ty| {
+            bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicIntegerType { span, name, ty });
+        };
+
+        let parse_atomic_ordering = |ord: ty::Value<'tcx>| {
+            let discr = ord.valtree.unwrap_branch()[0].unwrap_leaf();
+            discr.to_atomic_ordering()
+        };
 
         let llval = match name {
             sym::abort => {
@@ -105,6 +104,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 return Ok(());
             }
 
+            sym::caller_location => {
+                let location = self.get_caller_location(bx, source_info);
+                location.val.store(bx, result);
+                return Ok(());
+            }
+
             sym::va_start => bx.va_start(args[0].immediate()),
             sym::va_end => bx.va_end(args[0].immediate()),
             sym::size_of_val => {
@@ -126,7 +131,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     sym::vtable_align => ty::COMMON_VTABLE_ENTRIES_ALIGN,
                     _ => bug!(),
                 };
-                let value = meth::VirtualIndex::from_index(idx).get_usize(bx, vtable, callee_ty);
+                let value = meth::VirtualIndex::from_index(idx).get_usize(
+                    bx,
+                    vtable,
+                    instance.ty(bx.tcx(), bx.typing_env()),
+                );
                 match name {
                     // Size is always <= isize::MAX.
                     sym::vtable_size => {
@@ -141,13 +150,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
                 value
             }
-            sym::pref_align_of
-            | sym::needs_drop
-            | sym::type_id
-            | sym::type_name
-            | sym::variant_count => {
+            sym::needs_drop | sym::type_id | sym::type_name | sym::variant_count => {
                 let value = bx.tcx().const_eval_instance(bx.typing_env(), instance, span).unwrap();
-                OperandRef::from_const(bx, value, ret_ty).immediate_or_packed_pair(bx)
+                OperandRef::from_const(bx, value, result.layout.ty).immediate_or_packed_pair(bx)
             }
             sym::arith_offset => {
                 let ty = fn_args.type_at(0);
@@ -231,7 +236,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 bx.or_disjoint(a, b)
             }
             sym::exact_div => {
-                let ty = arg_tys[0];
+                let ty = args[0].layout.ty;
                 match int_type_width_signed(ty, bx.tcx()) {
                     Some((_width, signed)) => {
                         if signed {
@@ -251,7 +256,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
             }
             sym::fadd_fast | sym::fsub_fast | sym::fmul_fast | sym::fdiv_fast | sym::frem_fast => {
-                match float_type_width(arg_tys[0]) {
+                match float_type_width(args[0].layout.ty) {
                     Some(_width) => match name {
                         sym::fadd_fast => bx.fadd_fast(args[0].immediate(), args[1].immediate()),
                         sym::fsub_fast => bx.fsub_fast(args[0].immediate(), args[1].immediate()),
@@ -264,7 +269,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicFloatType {
                             span,
                             name,
-                            ty: arg_tys[0],
+                            ty: args[0].layout.ty,
                         });
                         return Ok(());
                     }
@@ -274,7 +279,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             | sym::fsub_algebraic
             | sym::fmul_algebraic
             | sym::fdiv_algebraic
-            | sym::frem_algebraic => match float_type_width(arg_tys[0]) {
+            | sym::frem_algebraic => match float_type_width(args[0].layout.ty) {
                 Some(_width) => match name {
                     sym::fadd_algebraic => {
                         bx.fadd_algebraic(args[0].immediate(), args[1].immediate())
@@ -297,199 +302,174 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicFloatType {
                         span,
                         name,
-                        ty: arg_tys[0],
+                        ty: args[0].layout.ty,
                     });
                     return Ok(());
                 }
             },
 
             sym::float_to_int_unchecked => {
-                if float_type_width(arg_tys[0]).is_none() {
+                if float_type_width(args[0].layout.ty).is_none() {
                     bx.tcx().dcx().emit_err(InvalidMonomorphization::FloatToIntUnchecked {
                         span,
-                        ty: arg_tys[0],
+                        ty: args[0].layout.ty,
                     });
                     return Ok(());
                 }
-                let Some((_width, signed)) = int_type_width_signed(ret_ty, bx.tcx()) else {
+                let Some((_width, signed)) = int_type_width_signed(result.layout.ty, bx.tcx())
+                else {
                     bx.tcx().dcx().emit_err(InvalidMonomorphization::FloatToIntUnchecked {
                         span,
-                        ty: ret_ty,
+                        ty: result.layout.ty,
                     });
                     return Ok(());
                 };
                 if signed {
-                    bx.fptosi(args[0].immediate(), llret_ty)
+                    bx.fptosi(args[0].immediate(), bx.backend_type(result.layout))
                 } else {
-                    bx.fptoui(args[0].immediate(), llret_ty)
+                    bx.fptoui(args[0].immediate(), bx.backend_type(result.layout))
                 }
             }
 
-            // This requires that atomic intrinsics follow a specific naming pattern:
-            // "atomic_<operation>[_<ordering>]"
-            name if let Some(atomic) = name_str.strip_prefix("atomic_") => {
-                use crate::common::AtomicOrdering::*;
-                use crate::common::{AtomicRmwBinOp, SynchronizationScope};
+            sym::atomic_load => {
+                let ty = fn_args.type_at(0);
+                if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) {
+                    invalid_monomorphization_int_type(ty);
+                    return Ok(());
+                }
+                let ordering = fn_args.const_at(1).to_value();
+                let layout = bx.layout_of(ty);
+                let source = args[0].immediate();
+                bx.atomic_load(
+                    bx.backend_type(layout),
+                    source,
+                    parse_atomic_ordering(ordering),
+                    layout.size,
+                )
+            }
+            sym::atomic_store => {
+                let ty = fn_args.type_at(0);
+                if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) {
+                    invalid_monomorphization_int_type(ty);
+                    return Ok(());
+                }
+                let ordering = fn_args.const_at(1).to_value();
+                let size = bx.layout_of(ty).size;
+                let val = args[1].immediate();
+                let ptr = args[0].immediate();
+                bx.atomic_store(val, ptr, parse_atomic_ordering(ordering), size);
+                return Ok(());
+            }
+            sym::atomic_cxchg | sym::atomic_cxchgweak => {
+                let ty = fn_args.type_at(0);
+                if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) {
+                    invalid_monomorphization_int_type(ty);
+                    return Ok(());
+                }
+                let succ_ordering = fn_args.const_at(1).to_value();
+                let fail_ordering = fn_args.const_at(2).to_value();
+                let weak = name == sym::atomic_cxchgweak;
+                let dst = args[0].immediate();
+                let cmp = args[1].immediate();
+                let src = args[2].immediate();
+                let (val, success) = bx.atomic_cmpxchg(
+                    dst,
+                    cmp,
+                    src,
+                    parse_atomic_ordering(succ_ordering),
+                    parse_atomic_ordering(fail_ordering),
+                    weak,
+                );
+                let val = bx.from_immediate(val);
+                let success = bx.from_immediate(success);
 
-                let Some((instruction, ordering)) = atomic.split_once('_') else {
-                    bx.sess().dcx().emit_fatal(errors::MissingMemoryOrdering);
-                };
+                let dest = result.project_field(bx, 0);
+                bx.store_to_place(val, dest.val);
+                let dest = result.project_field(bx, 1);
+                bx.store_to_place(success, dest.val);
 
-                let parse_ordering = |bx: &Bx, s| match s {
-                    "relaxed" => Relaxed,
-                    "acquire" => Acquire,
-                    "release" => Release,
-                    "acqrel" => AcquireRelease,
-                    "seqcst" => SequentiallyConsistent,
-                    _ => bx.sess().dcx().emit_fatal(errors::UnknownAtomicOrdering),
+                return Ok(());
+            }
+            // These are all AtomicRMW ops
+            sym::atomic_max | sym::atomic_min => {
+                let atom_op = if name == sym::atomic_max {
+                    AtomicRmwBinOp::AtomicMax
+                } else {
+                    AtomicRmwBinOp::AtomicMin
                 };
 
-                let invalid_monomorphization = |ty| {
-                    bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
-                        span,
-                        name,
-                        ty,
-                    });
+                let ty = fn_args.type_at(0);
+                if matches!(ty.kind(), ty::Int(_)) {
+                    let ordering = fn_args.const_at(1).to_value();
+                    let ptr = args[0].immediate();
+                    let val = args[1].immediate();
+                    bx.atomic_rmw(atom_op, ptr, val, parse_atomic_ordering(ordering))
+                } else {
+                    invalid_monomorphization_int_type(ty);
+                    return Ok(());
+                }
+            }
+            sym::atomic_umax | sym::atomic_umin => {
+                let atom_op = if name == sym::atomic_umax {
+                    AtomicRmwBinOp::AtomicUMax
+                } else {
+                    AtomicRmwBinOp::AtomicUMin
                 };
 
-                match instruction {
-                    "cxchg" | "cxchgweak" => {
-                        let Some((success, failure)) = ordering.split_once('_') else {
-                            bx.sess().dcx().emit_fatal(errors::AtomicCompareExchange);
-                        };
-                        let ty = fn_args.type_at(0);
-                        if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
-                            let weak = instruction == "cxchgweak";
-                            let dst = args[0].immediate();
-                            let cmp = args[1].immediate();
-                            let src = args[2].immediate();
-                            let (val, success) = bx.atomic_cmpxchg(
-                                dst,
-                                cmp,
-                                src,
-                                parse_ordering(bx, success),
-                                parse_ordering(bx, failure),
-                                weak,
-                            );
-                            let val = bx.from_immediate(val);
-                            let success = bx.from_immediate(success);
-
-                            let dest = result.project_field(bx, 0);
-                            bx.store_to_place(val, dest.val);
-                            let dest = result.project_field(bx, 1);
-                            bx.store_to_place(success, dest.val);
-                        } else {
-                            invalid_monomorphization(ty);
-                        }
-                        return Ok(());
-                    }
-
-                    "load" => {
-                        let ty = fn_args.type_at(0);
-                        if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
-                            let layout = bx.layout_of(ty);
-                            let size = layout.size;
-                            let source = args[0].immediate();
-                            bx.atomic_load(
-                                bx.backend_type(layout),
-                                source,
-                                parse_ordering(bx, ordering),
-                                size,
-                            )
-                        } else {
-                            invalid_monomorphization(ty);
-                            return Ok(());
-                        }
-                    }
-
-                    "store" => {
-                        let ty = fn_args.type_at(0);
-                        if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
-                            let size = bx.layout_of(ty).size;
-                            let val = args[1].immediate();
-                            let ptr = args[0].immediate();
-                            bx.atomic_store(val, ptr, parse_ordering(bx, ordering), size);
-                        } else {
-                            invalid_monomorphization(ty);
-                        }
-                        return Ok(());
-                    }
-
-                    "fence" => {
-                        bx.atomic_fence(
-                            parse_ordering(bx, ordering),
-                            SynchronizationScope::CrossThread,
-                        );
-                        return Ok(());
-                    }
-
-                    "singlethreadfence" => {
-                        bx.atomic_fence(
-                            parse_ordering(bx, ordering),
-                            SynchronizationScope::SingleThread,
-                        );
-                        return Ok(());
-                    }
-
-                    // These are all AtomicRMW ops
-                    "max" | "min" => {
-                        let atom_op = if instruction == "max" {
-                            AtomicRmwBinOp::AtomicMax
-                        } else {
-                            AtomicRmwBinOp::AtomicMin
-                        };
-
-                        let ty = fn_args.type_at(0);
-                        if matches!(ty.kind(), ty::Int(_)) {
-                            let ptr = args[0].immediate();
-                            let val = args[1].immediate();
-                            bx.atomic_rmw(atom_op, ptr, val, parse_ordering(bx, ordering))
-                        } else {
-                            invalid_monomorphization(ty);
-                            return Ok(());
-                        }
-                    }
-                    "umax" | "umin" => {
-                        let atom_op = if instruction == "umax" {
-                            AtomicRmwBinOp::AtomicUMax
-                        } else {
-                            AtomicRmwBinOp::AtomicUMin
-                        };
-
-                        let ty = fn_args.type_at(0);
-                        if matches!(ty.kind(), ty::Uint(_)) {
-                            let ptr = args[0].immediate();
-                            let val = args[1].immediate();
-                            bx.atomic_rmw(atom_op, ptr, val, parse_ordering(bx, ordering))
-                        } else {
-                            invalid_monomorphization(ty);
-                            return Ok(());
-                        }
-                    }
-                    op => {
-                        let atom_op = match op {
-                            "xchg" => AtomicRmwBinOp::AtomicXchg,
-                            "xadd" => AtomicRmwBinOp::AtomicAdd,
-                            "xsub" => AtomicRmwBinOp::AtomicSub,
-                            "and" => AtomicRmwBinOp::AtomicAnd,
-                            "nand" => AtomicRmwBinOp::AtomicNand,
-                            "or" => AtomicRmwBinOp::AtomicOr,
-                            "xor" => AtomicRmwBinOp::AtomicXor,
-                            _ => bx.sess().dcx().emit_fatal(errors::UnknownAtomicOperation),
-                        };
+                let ty = fn_args.type_at(0);
+                if matches!(ty.kind(), ty::Uint(_)) {
+                    let ordering = fn_args.const_at(1).to_value();
+                    let ptr = args[0].immediate();
+                    let val = args[1].immediate();
+                    bx.atomic_rmw(atom_op, ptr, val, parse_atomic_ordering(ordering))
+                } else {
+                    invalid_monomorphization_int_type(ty);
+                    return Ok(());
+                }
+            }
+            sym::atomic_xchg
+            | sym::atomic_xadd
+            | sym::atomic_xsub
+            | sym::atomic_and
+            | sym::atomic_nand
+            | sym::atomic_or
+            | sym::atomic_xor => {
+                let atom_op = match name {
+                    sym::atomic_xchg => AtomicRmwBinOp::AtomicXchg,
+                    sym::atomic_xadd => AtomicRmwBinOp::AtomicAdd,
+                    sym::atomic_xsub => AtomicRmwBinOp::AtomicSub,
+                    sym::atomic_and => AtomicRmwBinOp::AtomicAnd,
+                    sym::atomic_nand => AtomicRmwBinOp::AtomicNand,
+                    sym::atomic_or => AtomicRmwBinOp::AtomicOr,
+                    sym::atomic_xor => AtomicRmwBinOp::AtomicXor,
+                    _ => unreachable!(),
+                };
 
-                        let ty = fn_args.type_at(0);
-                        if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
-                            let ptr = args[0].immediate();
-                            let val = args[1].immediate();
-                            bx.atomic_rmw(atom_op, ptr, val, parse_ordering(bx, ordering))
-                        } else {
-                            invalid_monomorphization(ty);
-                            return Ok(());
-                        }
-                    }
+                let ty = fn_args.type_at(0);
+                if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
+                    let ordering = fn_args.const_at(1).to_value();
+                    let ptr = args[0].immediate();
+                    let val = args[1].immediate();
+                    bx.atomic_rmw(atom_op, ptr, val, parse_atomic_ordering(ordering))
+                } else {
+                    invalid_monomorphization_int_type(ty);
+                    return Ok(());
                 }
             }
+            sym::atomic_fence => {
+                let ordering = fn_args.const_at(0).to_value();
+                bx.atomic_fence(parse_atomic_ordering(ordering), SynchronizationScope::CrossThread);
+                return Ok(());
+            }
+
+            sym::atomic_singlethreadfence => {
+                let ordering = fn_args.const_at(0).to_value();
+                bx.atomic_fence(
+                    parse_atomic_ordering(ordering),
+                    SynchronizationScope::SingleThread,
+                );
+                return Ok(());
+            }
 
             sym::nontemporal_store => {
                 let dst = args[0].deref(bx.cx());
@@ -528,18 +508,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
             _ => {
                 // Need to use backend-specific things in the implementation.
-                return bx.codegen_intrinsic_call(instance, fn_abi, args, llresult, span);
+                return bx.codegen_intrinsic_call(instance, args, result, span);
             }
         };
 
-        if !fn_abi.ret.is_ignore() {
-            if let PassMode::Cast { .. } = &fn_abi.ret.mode {
-                bx.store_to_place(llval, result.val);
-            } else {
-                OperandRef::from_immediate_or_packed_pair(bx, llval, result.layout)
-                    .val
-                    .store(bx, result);
-            }
+        if result.layout.ty.is_bool() {
+            let val = bx.from_immediate(llval);
+            bx.store_to_place(val, result.val);
+        } else if !result.layout.ty.is_unit() {
+            bx.store_to_place(llval, result.val);
         }
         Ok(())
     }
diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs
index eade9e52de9..e9389ddf93b 100644
--- a/compiler/rustc_codegen_ssa/src/mir/operand.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs
@@ -45,9 +45,15 @@ pub enum OperandValue<V> {
     Immediate(V),
     /// A pair of immediate LLVM values. Used by wide pointers too.
     ///
-    /// An `OperandValue` *must* be this variant for any type for which
+    /// # Invariants
+    /// - For `Pair(a, b)`, `a` is always at offset 0, but may have `FieldIdx(1..)`
+    /// - `b` is not at offset 0, because `V` is not a 1ZST type.
+    /// - `a` and `b` will have a different FieldIdx, but otherwise `b`'s may be lower
+    ///   or they may not be adjacent, due to arbitrary numbers of 1ZST fields that
+    ///   will not affect the shape of the data which determines if `Pair` will be used.
+    /// - An `OperandValue` *must* be this variant for any type for which
     /// [`LayoutTypeCodegenMethods::is_backend_scalar_pair`] returns `true`.
-    /// The backend values in this variant must be the *immediate* backend types,
+    /// - The backend values in this variant must be the *immediate* backend types,
     /// as returned by [`LayoutTypeCodegenMethods::scalar_pair_element_backend_type`]
     /// with `immediate: true`.
     Pair(V, V),
@@ -462,10 +468,10 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
         let tag_op = match self.val {
             OperandValue::ZeroSized => bug!(),
             OperandValue::Immediate(_) | OperandValue::Pair(_, _) => {
-                self.extract_field(fx, bx, tag_field)
+                self.extract_field(fx, bx, tag_field.as_usize())
             }
             OperandValue::Ref(place) => {
-                let tag = place.with_type(self.layout).project_field(bx, tag_field);
+                let tag = place.with_type(self.layout).project_field(bx, tag_field.as_usize());
                 bx.load_operand(tag)
             }
         };
diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs
index 31db7fa9a18..937063c24a6 100644
--- a/compiler/rustc_codegen_ssa/src/mir/place.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/place.rs
@@ -250,7 +250,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
             Variants::Single { index } => assert_eq!(index, variant_index),
 
             Variants::Multiple { tag_encoding: TagEncoding::Direct, tag_field, .. } => {
-                let ptr = self.project_field(bx, tag_field);
+                let ptr = self.project_field(bx, tag_field.as_usize());
                 let to =
                     self.layout.ty.discriminant_for_variant(bx.tcx(), variant_index).unwrap().val;
                 bx.store_to_place(
@@ -265,7 +265,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
                 ..
             } => {
                 if variant_index != untagged_variant {
-                    let niche = self.project_field(bx, tag_field);
+                    let niche = self.project_field(bx, tag_field.as_usize());
                     let niche_llty = bx.cx().immediate_backend_type(niche.layout);
                     let BackendRepr::Scalar(scalar) = niche.layout.backend_repr else {
                         bug!("expected a scalar placeref for the niche");
diff --git a/compiler/rustc_codegen_ssa/src/mono_item.rs b/compiler/rustc_codegen_ssa/src/mono_item.rs
index c2067e52afe..7b4268abe4b 100644
--- a/compiler/rustc_codegen_ssa/src/mono_item.rs
+++ b/compiler/rustc_codegen_ssa/src/mono_item.rs
@@ -11,11 +11,13 @@ pub trait MonoItemExt<'a, 'tcx> {
     fn define<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
         cx: &'a mut Bx::CodegenCx,
+        cgu_name: &str,
         item_data: MonoItemData,
     );
     fn predefine<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        cx: &'a Bx::CodegenCx,
+        cx: &'a mut Bx::CodegenCx,
+        cgu_name: &str,
         linkage: Linkage,
         visibility: Visibility,
     );
@@ -26,14 +28,10 @@ impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> {
     fn define<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
         cx: &'a mut Bx::CodegenCx,
+        cgu_name: &str,
         item_data: MonoItemData,
     ) {
-        debug!(
-            "BEGIN IMPLEMENTING '{} ({})' in cgu {}",
-            self,
-            self.to_raw_string(),
-            cx.codegen_unit().name()
-        );
+        debug!("BEGIN IMPLEMENTING '{} ({})' in cgu {}", self, self.to_raw_string(), cgu_name);
 
         match *self {
             MonoItem::Static(def_id) => {
@@ -56,26 +54,17 @@ impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> {
             }
         }
 
-        debug!(
-            "END IMPLEMENTING '{} ({})' in cgu {}",
-            self,
-            self.to_raw_string(),
-            cx.codegen_unit().name()
-        );
+        debug!("END IMPLEMENTING '{} ({})' in cgu {}", self, self.to_raw_string(), cgu_name);
     }
 
     fn predefine<Bx: BuilderMethods<'a, 'tcx>>(
         &self,
-        cx: &'a Bx::CodegenCx,
+        cx: &'a mut Bx::CodegenCx,
+        cgu_name: &str,
         linkage: Linkage,
         visibility: Visibility,
     ) {
-        debug!(
-            "BEGIN PREDEFINING '{} ({})' in cgu {}",
-            self,
-            self.to_raw_string(),
-            cx.codegen_unit().name()
-        );
+        debug!("BEGIN PREDEFINING '{} ({})' in cgu {}", self, self.to_raw_string(), cgu_name);
 
         let symbol_name = self.symbol_name(cx.tcx()).name;
 
@@ -97,12 +86,7 @@ impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> {
             MonoItem::GlobalAsm(..) => {}
         }
 
-        debug!(
-            "END PREDEFINING '{} ({})' in cgu {}",
-            self,
-            self.to_raw_string(),
-            cx.codegen_unit().name()
-        );
+        debug!("END PREDEFINING '{} ({})' in cgu {}", self, self.to_raw_string(), cgu_name);
     }
 
     fn to_raw_string(&self) -> String {
diff --git a/compiler/rustc_codegen_ssa/src/size_of_val.rs b/compiler/rustc_codegen_ssa/src/size_of_val.rs
index ac2366340fb..577012151e4 100644
--- a/compiler/rustc_codegen_ssa/src/size_of_val.rs
+++ b/compiler/rustc_codegen_ssa/src/size_of_val.rs
@@ -5,6 +5,7 @@ use rustc_hir::LangItem;
 use rustc_middle::bug;
 use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
 use rustc_middle::ty::{self, Ty};
+use rustc_span::DUMMY_SP;
 use tracing::{debug, trace};
 
 use crate::common::IntPredicate;
@@ -62,7 +63,7 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 
             // Obtain the panic entry point.
             let (fn_abi, llfn, _instance) =
-                common::build_langcall(bx, None, LangItem::PanicNounwind);
+                common::build_langcall(bx, DUMMY_SP, LangItem::PanicNounwind);
 
             // Generate the call. Cannot use `do_call` since we don't have a MIR terminator so we
             // can't create a `TerminationCodegenHelper`. (But we are in good company, this code is
diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs
index 3ecea522837..6bb3150c1c5 100644
--- a/compiler/rustc_codegen_ssa/src/target_features.rs
+++ b/compiler/rustc_codegen_ssa/src/target_features.rs
@@ -8,6 +8,7 @@ use rustc_hir::def_id::{DefId, LOCAL_CRATE, LocalDefId};
 use rustc_middle::middle::codegen_fn_attrs::TargetFeature;
 use rustc_middle::query::Providers;
 use rustc_middle::ty::TyCtxt;
+use rustc_session::lint::builtin::AARCH64_SOFTFLOAT_NEON;
 use rustc_session::parse::feature_err;
 use rustc_span::{Span, Symbol, sym};
 use rustc_target::target_features::{self, Stability};
@@ -18,6 +19,7 @@ use crate::errors;
 /// Enabled target features are added to `target_features`.
 pub(crate) fn from_target_feature_attr(
     tcx: TyCtxt<'_>,
+    did: LocalDefId,
     attr: &hir::Attribute,
     rust_target_features: &UnordMap<String, target_features::Stability>,
     target_features: &mut Vec<TargetFeature>,
@@ -92,11 +94,22 @@ pub(crate) fn from_target_feature_attr(
                     // generating code so "it's fine".
                     if !tcx.sess.opts.actually_rustdoc {
                         if abi_feature_constraints.incompatible.contains(&name.as_str()) {
-                            tcx.dcx().emit_err(errors::ForbiddenTargetFeatureAttr {
-                                span: item.span(),
-                                feature: name.as_str(),
-                                reason: "this feature is incompatible with the target ABI",
-                            });
+                            // For "neon" specifically, we emit an FCW instead of a hard error.
+                            // See <https://github.com/rust-lang/rust/issues/134375>.
+                            if tcx.sess.target.arch == "aarch64" && name.as_str() == "neon" {
+                                tcx.emit_node_span_lint(
+                                    AARCH64_SOFTFLOAT_NEON,
+                                    tcx.local_def_id_to_hir_id(did),
+                                    item.span(),
+                                    errors::Aarch64SoftfloatNeon,
+                                );
+                            } else {
+                                tcx.dcx().emit_err(errors::ForbiddenTargetFeatureAttr {
+                                    span: item.span(),
+                                    feature: name.as_str(),
+                                    reason: "this feature is incompatible with the target ABI",
+                                });
+                            }
                         }
                     }
                     target_features.push(TargetFeature { name, implied: name != feature_sym })
diff --git a/compiler/rustc_codegen_ssa/src/traits/backend.rs b/compiler/rustc_codegen_ssa/src/traits/backend.rs
index e2f1458d062..95bf3b16685 100644
--- a/compiler/rustc_codegen_ssa/src/traits/backend.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/backend.rs
@@ -21,12 +21,12 @@ use crate::back::write::TargetMachineFactoryFn;
 use crate::{CodegenResults, ModuleCodegen, TargetConfig};
 
 pub trait BackendTypes {
-    type Value: CodegenObject;
+    type Value: CodegenObject + PartialEq;
     type Metadata: CodegenObject;
     type Function: CodegenObject;
 
     type BasicBlock: Copy;
-    type Type: CodegenObject;
+    type Type: CodegenObject + PartialEq;
     type Funclet;
 
     // FIXME(eddyb) find a common convention for all of the debuginfo-related
@@ -97,13 +97,6 @@ pub trait CodegenBackend {
     fn link(&self, sess: &Session, codegen_results: CodegenResults, outputs: &OutputFilenames) {
         link_binary(sess, &ArArchiveBuilderBuilder, codegen_results, outputs);
     }
-
-    /// Returns `true` if this backend can be safely called from multiple threads.
-    ///
-    /// Defaults to `true`.
-    fn supports_parallel(&self) -> bool {
-        true
-    }
 }
 
 pub trait ExtraBackendMethods:
@@ -144,4 +137,11 @@ pub trait ExtraBackendMethods:
     {
         std::thread::Builder::new().name(name).spawn(f)
     }
+
+    /// Returns `true` if this backend can be safely called from multiple threads.
+    ///
+    /// Defaults to `true`.
+    fn supports_parallel(&self) -> bool {
+        true
+    }
 }
diff --git a/compiler/rustc_codegen_ssa/src/traits/builder.rs b/compiler/rustc_codegen_ssa/src/traits/builder.rs
index f66309cf340..7f78bc75695 100644
--- a/compiler/rustc_codegen_ssa/src/traits/builder.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/builder.rs
@@ -4,7 +4,7 @@ use std::ops::Deref;
 use rustc_abi::{Align, Scalar, Size, WrappingRange};
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
 use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout};
-use rustc_middle::ty::{Instance, Ty};
+use rustc_middle::ty::{AtomicOrdering, Instance, Ty};
 use rustc_session::config::OptLevel;
 use rustc_span::Span;
 use rustc_target::callconv::FnAbi;
@@ -19,9 +19,7 @@ use super::misc::MiscCodegenMethods;
 use super::type_::{ArgAbiBuilderMethods, BaseTypeCodegenMethods, LayoutTypeCodegenMethods};
 use super::{CodegenMethods, StaticBuilderMethods};
 use crate::MemFlags;
-use crate::common::{
-    AtomicOrdering, AtomicRmwBinOp, IntPredicate, RealPredicate, SynchronizationScope, TypeKind,
-};
+use crate::common::{AtomicRmwBinOp, IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
 use crate::mir::operand::{OperandRef, OperandValue};
 use crate::mir::place::{PlaceRef, PlaceValue};
 
@@ -514,11 +512,11 @@ pub trait BuilderMethods<'a, 'tcx>:
     fn extract_value(&mut self, agg_val: Self::Value, idx: u64) -> Self::Value;
     fn insert_value(&mut self, agg_val: Self::Value, elt: Self::Value, idx: u64) -> Self::Value;
 
-    fn set_personality_fn(&mut self, personality: Self::Value);
+    fn set_personality_fn(&mut self, personality: Self::Function);
 
     // These are used by everyone except msvc
-    fn cleanup_landing_pad(&mut self, pers_fn: Self::Value) -> (Self::Value, Self::Value);
-    fn filter_landing_pad(&mut self, pers_fn: Self::Value) -> (Self::Value, Self::Value);
+    fn cleanup_landing_pad(&mut self, pers_fn: Self::Function) -> (Self::Value, Self::Value);
+    fn filter_landing_pad(&mut self, pers_fn: Self::Function) -> (Self::Value, Self::Value);
     fn resume(&mut self, exn0: Self::Value, exn1: Self::Value);
 
     // These are used only by msvc
diff --git a/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs b/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs
index 30d77c206a5..b9d4950e0ad 100644
--- a/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/debuginfo.rs
@@ -81,7 +81,6 @@ pub trait DebugInfoBuilderMethods: BackendTypes {
     );
     fn set_dbg_loc(&mut self, dbg_loc: Self::DILocation);
     fn clear_dbg_loc(&mut self);
-    fn get_dbg_loc(&self) -> Option<Self::DILocation>;
     fn insert_reference_to_gdb_debug_scripts_section_global(&mut self);
     fn set_var_name(&mut self, value: Self::Value, name: &str);
 }
diff --git a/compiler/rustc_codegen_ssa/src/traits/declare.rs b/compiler/rustc_codegen_ssa/src/traits/declare.rs
index c1edeac31b0..9f735546558 100644
--- a/compiler/rustc_codegen_ssa/src/traits/declare.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/declare.rs
@@ -4,14 +4,14 @@ use rustc_middle::ty::Instance;
 
 pub trait PreDefineCodegenMethods<'tcx> {
     fn predefine_static(
-        &self,
+        &mut self,
         def_id: DefId,
         linkage: Linkage,
         visibility: Visibility,
         symbol_name: &str,
     );
     fn predefine_fn(
-        &self,
+        &mut self,
         instance: Instance<'tcx>,
         linkage: Linkage,
         visibility: Visibility,
diff --git a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs
index 88cf8dbf0c5..7d0c6be4c65 100644
--- a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs
@@ -1,9 +1,9 @@
-use rustc_middle::ty::{self, Ty};
+use rustc_middle::ty;
 use rustc_span::Span;
-use rustc_target::callconv::FnAbi;
 
 use super::BackendTypes;
 use crate::mir::operand::OperandRef;
+use crate::mir::place::PlaceRef;
 
 pub trait IntrinsicCallBuilderMethods<'tcx>: BackendTypes {
     /// Remember to add all intrinsics here, in `compiler/rustc_hir_analysis/src/check/mod.rs`,
@@ -14,17 +14,14 @@ pub trait IntrinsicCallBuilderMethods<'tcx>: BackendTypes {
     fn codegen_intrinsic_call(
         &mut self,
         instance: ty::Instance<'tcx>,
-        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
         args: &[OperandRef<'tcx, Self::Value>],
-        llresult: Self::Value,
+        result: PlaceRef<'tcx, Self::Value>,
         span: Span,
     ) -> Result<(), ty::Instance<'tcx>>;
 
     fn abort(&mut self);
     fn assume(&mut self, val: Self::Value);
     fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value;
-    /// Trait method used to test whether a given pointer is associated with a type identifier.
-    fn type_test(&mut self, pointer: Self::Value, typeid: Self::Metadata) -> Self::Value;
     /// Trait method used to load a function while testing if it is associated with a type
     /// identifier.
     fn type_checked_load(
diff --git a/compiler/rustc_codegen_ssa/src/traits/misc.rs b/compiler/rustc_codegen_ssa/src/traits/misc.rs
index 4004947b464..710fab27901 100644
--- a/compiler/rustc_codegen_ssa/src/traits/misc.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/misc.rs
@@ -1,7 +1,6 @@
 use std::cell::RefCell;
 
 use rustc_data_structures::fx::FxHashMap;
-use rustc_middle::mir::mono::CodegenUnit;
 use rustc_middle::ty::{self, Instance, Ty};
 use rustc_session::Session;
 
@@ -20,9 +19,8 @@ pub trait MiscCodegenMethods<'tcx>: BackendTypes {
     }
     fn get_fn(&self, instance: Instance<'tcx>) -> Self::Function;
     fn get_fn_addr(&self, instance: Instance<'tcx>) -> Self::Value;
-    fn eh_personality(&self) -> Self::Value;
+    fn eh_personality(&self) -> Self::Function;
     fn sess(&self) -> &Session;
-    fn codegen_unit(&self) -> &'tcx CodegenUnit<'tcx>;
     fn set_frame_pointer_type(&self, llfn: Self::Function);
     fn apply_target_cpu_attr(&self, llfn: Self::Function);
     /// Declares the extern "C" main function for the entry point. Returns None if the symbol
diff --git a/compiler/rustc_codegen_ssa/src/traits/mod.rs b/compiler/rustc_codegen_ssa/src/traits/mod.rs
index 239857a4298..6d1ac717c0b 100644
--- a/compiler/rustc_codegen_ssa/src/traits/mod.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/mod.rs
@@ -50,7 +50,7 @@ pub use self::type_::{
 };
 pub use self::write::{ModuleBufferMethods, ThinBufferMethods, WriteBackendMethods};
 
-pub trait CodegenObject = Copy + PartialEq + fmt::Debug;
+pub trait CodegenObject = Copy + fmt::Debug;
 
 pub trait CodegenMethods<'tcx> = LayoutOf<'tcx, LayoutOfResult = TyAndLayout<'tcx>>
     + FnAbiOf<'tcx, FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>>
diff --git a/compiler/rustc_codegen_ssa/src/traits/statics.rs b/compiler/rustc_codegen_ssa/src/traits/statics.rs
index ece0ea1b2ea..0e1e445c72f 100644
--- a/compiler/rustc_codegen_ssa/src/traits/statics.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/statics.rs
@@ -5,19 +5,7 @@ use super::BackendTypes;
 
 pub trait StaticCodegenMethods: BackendTypes {
     fn static_addr_of(&self, cv: Self::Value, align: Align, kind: Option<&str>) -> Self::Value;
-    fn codegen_static(&self, def_id: DefId);
-
-    /// Mark the given global value as "used", to prevent the compiler and linker from potentially
-    /// removing a static variable that may otherwise appear unused.
-    fn add_used_global(&self, global: Self::Value);
-
-    /// Same as add_used_global(), but only prevent the compiler from potentially removing an
-    /// otherwise unused symbol. The linker is still permitted to drop it.
-    ///
-    /// This corresponds to the documented semantics of the `#[used]` attribute, although
-    /// on some targets (non-ELF), we may use `add_used_global` for `#[used]` statics
-    /// instead.
-    fn add_compiler_used_global(&self, global: Self::Value);
+    fn codegen_static(&mut self, def_id: DefId);
 }
 
 pub trait StaticBuilderMethods: BackendTypes {
diff --git a/compiler/rustc_codegen_ssa/src/traits/type_.rs b/compiler/rustc_codegen_ssa/src/traits/type_.rs
index 32d9f27d32d..c3fc21a9285 100644
--- a/compiler/rustc_codegen_ssa/src/traits/type_.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/type_.rs
@@ -158,7 +158,6 @@ pub trait ArgAbiBuilderMethods<'tcx>: BackendTypes {
         val: Self::Value,
         dst: PlaceRef<'tcx, Self::Value>,
     );
-    fn arg_memory_ty(&self, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>) -> Self::Type;
 }
 
 pub trait TypeCodegenMethods<'tcx> = DerivedTypeCodegenMethods<'tcx>
diff --git a/compiler/rustc_codegen_ssa/src/traits/write.rs b/compiler/rustc_codegen_ssa/src/traits/write.rs
index c77efdd1728..07a0609fda1 100644
--- a/compiler/rustc_codegen_ssa/src/traits/write.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/write.rs
@@ -6,7 +6,7 @@ use crate::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule};
 use crate::back::write::{CodegenContext, FatLtoInput, ModuleConfig};
 use crate::{CompiledModule, ModuleCodegen};
 
-pub trait WriteBackendMethods: 'static + Sized + Clone {
+pub trait WriteBackendMethods: Clone + 'static {
     type Module: Send + Sync;
     type TargetMachine;
     type TargetMachineError;
@@ -37,7 +37,7 @@ pub trait WriteBackendMethods: 'static + Sized + Clone {
     ) -> Result<(Vec<LtoModuleCodegen<Self>>, Vec<WorkProduct>), FatalError>;
     fn print_pass_timings(&self);
     fn print_statistics(&self);
-    unsafe fn optimize(
+    fn optimize(
         cgcx: &CodegenContext<Self>,
         dcx: DiagCtxtHandle<'_>,
         module: &mut ModuleCodegen<Self::Module>,
@@ -47,11 +47,11 @@ pub trait WriteBackendMethods: 'static + Sized + Clone {
         cgcx: &CodegenContext<Self>,
         llmod: &mut ModuleCodegen<Self::Module>,
     ) -> Result<(), FatalError>;
-    unsafe fn optimize_thin(
+    fn optimize_thin(
         cgcx: &CodegenContext<Self>,
         thin: ThinModule<Self>,
     ) -> Result<ModuleCodegen<Self::Module>, FatalError>;
-    unsafe fn codegen(
+    fn codegen(
         cgcx: &CodegenContext<Self>,
         dcx: DiagCtxtHandle<'_>,
         module: ModuleCodegen<Self::Module>,
diff --git a/compiler/rustc_const_eval/messages.ftl b/compiler/rustc_const_eval/messages.ftl
index f4defd2aa13..7f9abe8aa8e 100644
--- a/compiler/rustc_const_eval/messages.ftl
+++ b/compiler/rustc_const_eval/messages.ftl
@@ -88,11 +88,9 @@ const_eval_division_overflow =
 const_eval_dyn_call_not_a_method =
     `dyn` call trying to call something that is not a method
 
-const_eval_error = {$error_kind ->
-    [static] could not evaluate static initializer
-    [const] evaluation of constant value failed
-    [const_with_path] evaluation of `{$instance}` failed
-    *[other] {""}
+const_eval_error = evaluation of `{$instance}` failed {$num_frames ->
+    [0] here
+    *[other] inside this call
 }
 
 const_eval_exact_div_has_remainder =
@@ -118,7 +116,7 @@ const_eval_frame_note_inner = inside {$where_ ->
 const_eval_frame_note_last = the failure occurred here
 
 const_eval_incompatible_calling_conventions =
-    calling a function with calling convention {$callee_conv} using calling convention {$caller_conv}
+    calling a function with calling convention "{$callee_conv}" using calling convention "{$caller_conv}"
 
 const_eval_incompatible_return_types =
     calling a function with return type {$callee_ty} passing return place of type {$caller_ty}
@@ -424,8 +422,7 @@ const_eval_unstable_in_stable_exposed =
     .unstable_sugg = if the {$is_function_call2 ->
             [true] caller
             *[false] function
-        } is not (yet) meant to be exposed to stable, add `#[rustc_const_unstable]` (this is what you probably want to do)
-    .bypass_sugg = otherwise, as a last resort `#[rustc_allow_const_fn_unstable]` can be used to bypass stability checks (this requires team approval)
+        } is not (yet) meant to be exposed to stable const contexts, add `#[rustc_const_unstable]`
 
 const_eval_unstable_intrinsic = `{$name}` is not yet stable as a const intrinsic
 const_eval_unstable_intrinsic_suggestion = add `#![feature({$feature})]` to the crate attributes to enable
diff --git a/compiler/rustc_const_eval/src/check_consts/check.rs b/compiler/rustc_const_eval/src/check_consts/check.rs
index b67a3ce03a9..6167f8cd4b5 100644
--- a/compiler/rustc_const_eval/src/check_consts/check.rs
+++ b/compiler/rustc_const_eval/src/check_consts/check.rs
@@ -589,12 +589,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
 
             Rvalue::Aggregate(kind, ..) => {
                 if let AggregateKind::Coroutine(def_id, ..) = kind.as_ref()
-                    && let Some(
-                        coroutine_kind @ hir::CoroutineKind::Desugared(
-                            hir::CoroutineDesugaring::Async,
-                            _,
-                        ),
-                    ) = self.tcx.coroutine_kind(def_id)
+                    && let Some(coroutine_kind) = self.tcx.coroutine_kind(def_id)
                 {
                     self.check_op(ops::Coroutine(coroutine_kind));
                 }
diff --git a/compiler/rustc_const_eval/src/check_consts/ops.rs b/compiler/rustc_const_eval/src/check_consts/ops.rs
index 1e5b98675c4..9c30dbff99e 100644
--- a/compiler/rustc_const_eval/src/check_consts/ops.rs
+++ b/compiler/rustc_const_eval/src/check_consts/ops.rs
@@ -281,7 +281,7 @@ fn build_error_for_const_call<'tcx>(
                 let mut sugg = None;
 
                 if ccx.tcx.is_lang_item(trait_id, LangItem::PartialEq) {
-                    match (args[0].unpack(), args[1].unpack()) {
+                    match (args[0].kind(), args[1].kind()) {
                         (GenericArgKind::Type(self_ty), GenericArgKind::Type(rhs_ty))
                             if self_ty == rhs_ty
                                 && self_ty.is_ref()
@@ -345,11 +345,7 @@ fn build_error_for_const_call<'tcx>(
                 non_or_conditionally,
             });
 
-            note_trait_if_possible(
-                &mut err,
-                self_ty,
-                tcx.require_lang_item(LangItem::Deref, Some(span)),
-            );
+            note_trait_if_possible(&mut err, self_ty, tcx.require_lang_item(LangItem::Deref, span));
             err
         }
         _ if tcx.opt_parent(callee) == tcx.get_diagnostic_item(sym::FmtArgumentsNew) => {
@@ -486,24 +482,25 @@ impl<'tcx> NonConstOp<'tcx> for IntrinsicUnstable {
 pub(crate) struct Coroutine(pub hir::CoroutineKind);
 impl<'tcx> NonConstOp<'tcx> for Coroutine {
     fn status_in_item(&self, _: &ConstCx<'_, 'tcx>) -> Status {
-        if let hir::CoroutineKind::Desugared(
-            hir::CoroutineDesugaring::Async,
-            hir::CoroutineSource::Block,
-        ) = self.0
-        {
-            Status::Unstable {
+        match self.0 {
+            hir::CoroutineKind::Desugared(
+                hir::CoroutineDesugaring::Async,
+                hir::CoroutineSource::Block,
+            )
+            // FIXME(coroutines): eventually we want to gate const coroutine coroutines behind a
+            // different feature.
+            | hir::CoroutineKind::Coroutine(_) => Status::Unstable {
                 gate: sym::const_async_blocks,
                 gate_already_checked: false,
                 safe_to_expose_on_stable: false,
                 is_function_call: false,
-            }
-        } else {
-            Status::Forbidden
+            },
+            _ => Status::Forbidden,
         }
     }
 
     fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> Diag<'tcx> {
-        let msg = format!("{:#}s are not allowed in {}s", self.0, ccx.const_kind());
+        let msg = format!("{} are not allowed in {}s", self.0.to_plural_string(), ccx.const_kind());
         if let Status::Unstable { gate, .. } = self.status_in_item(ccx) {
             ccx.tcx.sess.create_feature_err(errors::UnallowedOpInConstContext { span, msg }, gate)
         } else {
diff --git a/compiler/rustc_const_eval/src/check_consts/qualifs.rs b/compiler/rustc_const_eval/src/check_consts/qualifs.rs
index dfcd1969a73..c1a37ab6a83 100644
--- a/compiler/rustc_const_eval/src/check_consts/qualifs.rs
+++ b/compiler/rustc_const_eval/src/check_consts/qualifs.rs
@@ -99,7 +99,7 @@ impl Qualif for HasMutInterior {
         // requires borrowck, which in turn will invoke mir_const_qualifs again, causing a cycle error.
         // Instead we invoke an obligation context manually, and provide the opaque type inference settings
         // that allow the trait solver to just error out instead of cycling.
-        let freeze_def_id = cx.tcx.require_lang_item(LangItem::Freeze, Some(cx.body.span));
+        let freeze_def_id = cx.tcx.require_lang_item(LangItem::Freeze, cx.body.span);
         // FIXME(#132279): Once we've got a typing mode which reveals opaque types using the HIR
         // typeck results without causing query cycles, we should use this here instead of defining
         // opaque types.
@@ -180,7 +180,7 @@ impl Qualif for NeedsNonConstDrop {
         // that the components of this type are also `~const Destruct`. This
         // amounts to verifying that there are no values in this ADT that may have
         // a non-const drop.
-        let destruct_def_id = cx.tcx.require_lang_item(LangItem::Destruct, Some(cx.body.span));
+        let destruct_def_id = cx.tcx.require_lang_item(LangItem::Destruct, cx.body.span);
         let (infcx, param_env) = cx.tcx.infer_ctxt().build_with_typing_env(cx.typing_env);
         let ocx = ObligationCtxt::new(&infcx);
         ocx.register_obligation(Obligation::new(
diff --git a/compiler/rustc_const_eval/src/const_eval/dummy_machine.rs b/compiler/rustc_const_eval/src/const_eval/dummy_machine.rs
index 46dcebc46e9..b6e2682af36 100644
--- a/compiler/rustc_const_eval/src/const_eval/dummy_machine.rs
+++ b/compiler/rustc_const_eval/src/const_eval/dummy_machine.rs
@@ -90,7 +90,7 @@ impl<'tcx> interpret::Machine<'tcx> for DummyMachine {
         _instance: ty::Instance<'tcx>,
         _abi: &FnAbi<'tcx, Ty<'tcx>>,
         _args: &[interpret::FnArg<'tcx, Self::Provenance>],
-        _destination: &interpret::MPlaceTy<'tcx, Self::Provenance>,
+        _destination: &interpret::PlaceTy<'tcx, Self::Provenance>,
         _target: Option<BasicBlock>,
         _unwind: UnwindAction,
     ) -> interpret::InterpResult<'tcx, Option<(&'tcx Body<'tcx>, ty::Instance<'tcx>)>> {
@@ -108,7 +108,7 @@ impl<'tcx> interpret::Machine<'tcx> for DummyMachine {
         _ecx: &mut InterpCx<'tcx, Self>,
         _instance: ty::Instance<'tcx>,
         _args: &[interpret::OpTy<'tcx, Self::Provenance>],
-        _destination: &interpret::MPlaceTy<'tcx, Self::Provenance>,
+        _destination: &interpret::PlaceTy<'tcx, Self::Provenance>,
         _target: Option<BasicBlock>,
         _unwind: UnwindAction,
     ) -> interpret::InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
@@ -197,4 +197,9 @@ impl<'tcx> interpret::Machine<'tcx> for DummyMachine {
     ) -> &'a mut Vec<interpret::Frame<'tcx, Self::Provenance, Self::FrameExtra>> {
         unimplemented!()
     }
+
+    fn get_default_alloc_params(
+        &self,
+    ) -> <Self::Bytes as rustc_middle::mir::interpret::AllocBytes>::AllocParams {
+    }
 }
diff --git a/compiler/rustc_const_eval/src/const_eval/error.rs b/compiler/rustc_const_eval/src/const_eval/error.rs
index ffb32fa41eb..08fc03d9c46 100644
--- a/compiler/rustc_const_eval/src/const_eval/error.rs
+++ b/compiler/rustc_const_eval/src/const_eval/error.rs
@@ -1,6 +1,6 @@
 use std::mem;
 
-use rustc_errors::{DiagArgName, DiagArgValue, DiagMessage, Diagnostic, IntoDiagArg};
+use rustc_errors::{Diag, DiagArgName, DiagArgValue, DiagMessage, IntoDiagArg};
 use rustc_middle::mir::AssertKind;
 use rustc_middle::mir::interpret::{Provenance, ReportedErrorInfo};
 use rustc_middle::query::TyCtxtAt;
@@ -131,10 +131,10 @@ pub fn get_span_and_frames<'tcx>(
 
 /// Create a diagnostic for a const eval error.
 ///
-/// This will use the `mk` function for creating the error which will get passed labels according to
-/// the `InterpError` and the span and a stacktrace of current execution according to
-/// `get_span_and_frames`.
-pub(super) fn report<'tcx, C, F, E>(
+/// This will use the `mk` function for adding more information to the error.
+/// You can use it to add a stacktrace of current execution according to
+/// `get_span_and_frames` or just give context on where the const eval error happened.
+pub(super) fn report<'tcx, C, F>(
     tcx: TyCtxt<'tcx>,
     error: InterpErrorKind<'tcx>,
     span: Span,
@@ -143,8 +143,7 @@ pub(super) fn report<'tcx, C, F, E>(
 ) -> ErrorHandled
 where
     C: FnOnce() -> (Span, Vec<FrameNote>),
-    F: FnOnce(Span, Vec<FrameNote>) -> E,
-    E: Diagnostic<'tcx>,
+    F: FnOnce(&mut Diag<'_>, Span, Vec<FrameNote>),
 {
     // Special handling for certain errors
     match error {
@@ -163,8 +162,7 @@ where
         _ => {
             let (our_span, frames) = get_span_and_frames();
             let span = span.substitute_dummy(our_span);
-            let err = mk(span, frames);
-            let mut err = tcx.dcx().create_err(err);
+            let mut err = tcx.dcx().struct_span_err(our_span, error.diagnostic_message());
             // We allow invalid programs in infallible promoteds since invalid layouts can occur
             // anyway (e.g. due to size overflow). And we allow OOM as that can happen any time.
             let allowed_in_infallible = matches!(
@@ -172,11 +170,9 @@ where
                 InterpErrorKind::ResourceExhaustion(_) | InterpErrorKind::InvalidProgram(_)
             );
 
-            let msg = error.diagnostic_message();
             error.add_args(&mut err);
 
-            // Use *our* span to label the interp error
-            err.span_label(our_span, msg);
+            mk(&mut err, span, frames);
             let g = err.emit();
             let reported = if allowed_in_infallible {
                 ReportedErrorInfo::allowed_in_infallible(g)
diff --git a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
index ce8eceebdf8..be840191547 100644
--- a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
+++ b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
@@ -2,6 +2,7 @@ use std::sync::atomic::Ordering::Relaxed;
 
 use either::{Left, Right};
 use rustc_abi::{self as abi, BackendRepr};
+use rustc_errors::E0080;
 use rustc_hir::def::DefKind;
 use rustc_middle::mir::interpret::{AllocId, ErrorHandled, InterpErrorInfo, ReportedErrorInfo};
 use rustc_middle::mir::{self, ConstAlloc, ConstValue};
@@ -71,7 +72,12 @@ fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
 
     // This can't use `init_stack_frame` since `body` is not a function,
     // so computing its ABI would fail. It's also not worth it since there are no arguments to pass.
-    ecx.push_stack_frame_raw(cid.instance, body, &ret, StackPopCleanup::Root { cleanup: false })?;
+    ecx.push_stack_frame_raw(
+        cid.instance,
+        body,
+        &ret.clone().into(),
+        StackPopCleanup::Root { cleanup: false },
+    )?;
     ecx.storage_live_for_always_live_locals()?;
 
     // The main interpreter loop.
@@ -285,12 +291,18 @@ pub fn eval_to_const_value_raw_provider<'tcx>(
             |error| {
                 let span = tcx.def_span(def_id);
 
+                // FIXME(oli-obk): why don't we have any tests for this code path?
                 super::report(
                     tcx,
                     error.into_kind(),
                     span,
                     || (span, vec![]),
-                    |span, _| errors::NullaryIntrinsicError { span },
+                    |diag, span, _| {
+                        diag.span_label(
+                            span,
+                            crate::fluent_generated::const_eval_nullary_intrinsic_fail,
+                        );
+                    },
                 )
             },
         );
@@ -418,31 +430,24 @@ fn report_eval_error<'tcx>(
     let (error, backtrace) = error.into_parts();
     backtrace.print_backtrace();
 
-    let (kind, instance) = if ecx.tcx.is_static(cid.instance.def_id()) {
-        ("static", String::new())
-    } else {
-        // If the current item has generics, we'd like to enrich the message with the
-        // instance and its args: to show the actual compile-time values, in addition to
-        // the expression, leading to the const eval error.
-        let instance = &cid.instance;
-        if !instance.args.is_empty() {
-            let instance = with_no_trimmed_paths!(instance.to_string());
-            ("const_with_path", instance)
-        } else {
-            ("const", String::new())
-        }
-    };
+    let instance = with_no_trimmed_paths!(cid.instance.to_string());
 
     super::report(
         *ecx.tcx,
         error,
         DUMMY_SP,
         || super::get_span_and_frames(ecx.tcx, ecx.stack()),
-        |span, frames| errors::ConstEvalError {
-            span,
-            error_kind: kind,
-            instance,
-            frame_notes: frames,
+        |diag, span, frames| {
+            let num_frames = frames.len();
+            // FIXME(oli-obk): figure out how to use structured diagnostics again.
+            diag.code(E0080);
+            diag.span_label(span, crate::fluent_generated::const_eval_error);
+            for frame in frames {
+                diag.subdiagnostic(frame);
+            }
+            // Add after the frame rendering above, as it adds its own `instance` args.
+            diag.arg("instance", instance);
+            diag.arg("num_frames", num_frames);
         },
     )
 }
@@ -472,6 +477,15 @@ fn report_validation_error<'tcx>(
         error,
         DUMMY_SP,
         || crate::const_eval::get_span_and_frames(ecx.tcx, ecx.stack()),
-        move |span, frames| errors::ValidationFailure { span, ub_note: (), frames, raw_bytes },
+        move |diag, span, frames| {
+            // FIXME(oli-obk): figure out how to use structured diagnostics again.
+            diag.code(E0080);
+            diag.span_label(span, crate::fluent_generated::const_eval_validation_failure);
+            diag.note(crate::fluent_generated::const_eval_validation_failure_note);
+            for frame in frames {
+                diag.subdiagnostic(frame);
+            }
+            diag.subdiagnostic(raw_bytes);
+        },
     )
 }
diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs
index 7c7daed525b..a68dcf29988 100644
--- a/compiler/rustc_const_eval/src/const_eval/machine.rs
+++ b/compiler/rustc_const_eval/src/const_eval/machine.rs
@@ -22,7 +22,7 @@ use crate::errors::{LongRunning, LongRunningWarn};
 use crate::fluent_generated as fluent;
 use crate::interpret::{
     self, AllocId, AllocInit, AllocRange, ConstAllocation, CtfeProvenance, FnArg, Frame,
-    GlobalAlloc, ImmTy, InterpCx, InterpResult, MPlaceTy, OpTy, Pointer, RangeSet, Scalar,
+    GlobalAlloc, ImmTy, InterpCx, InterpResult, OpTy, PlaceTy, Pointer, RangeSet, Scalar,
     compile_time_machine, interp_ok, throw_exhaust, throw_inval, throw_ub, throw_ub_custom,
     throw_unsup, throw_unsup_format,
 };
@@ -226,7 +226,7 @@ impl<'tcx> CompileTimeInterpCx<'tcx> {
         &mut self,
         instance: ty::Instance<'tcx>,
         args: &[FnArg<'tcx>],
-        _dest: &MPlaceTy<'tcx>,
+        _dest: &PlaceTy<'tcx>,
         _ret: Option<mir::BasicBlock>,
     ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
         let def_id = instance.def_id();
@@ -249,7 +249,7 @@ impl<'tcx> CompileTimeInterpCx<'tcx> {
             return Err(ConstEvalErrKind::Panic { msg, file, line, col }).into();
         } else if self.tcx.is_lang_item(def_id, LangItem::PanicFmt) {
             // For panic_fmt, call const_panic_fmt instead.
-            let const_def_id = self.tcx.require_lang_item(LangItem::ConstPanicFmt, None);
+            let const_def_id = self.tcx.require_lang_item(LangItem::ConstPanicFmt, self.tcx.span);
             let new_instance = ty::Instance::expect_resolve(
                 *self.tcx,
                 self.typing_env(),
@@ -343,7 +343,7 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
         orig_instance: ty::Instance<'tcx>,
         _abi: &FnAbi<'tcx, Ty<'tcx>>,
         args: &[FnArg<'tcx>],
-        dest: &MPlaceTy<'tcx>,
+        dest: &PlaceTy<'tcx>,
         ret: Option<mir::BasicBlock>,
         _unwind: mir::UnwindAction, // unwinding is not supported in consts
     ) -> InterpResult<'tcx, Option<(&'tcx mir::Body<'tcx>, ty::Instance<'tcx>)>> {
@@ -385,7 +385,7 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
         ecx: &mut InterpCx<'tcx, Self>,
         instance: ty::Instance<'tcx>,
         args: &[OpTy<'tcx>],
-        dest: &MPlaceTy<'tcx, Self::Provenance>,
+        dest: &PlaceTy<'tcx, Self::Provenance>,
         target: Option<mir::BasicBlock>,
         _unwind: mir::UnwindAction,
     ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
@@ -735,6 +735,9 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
             Cow::Owned(compute_range())
         }
     }
+
+    fn get_default_alloc_params(&self) -> <Self::Bytes as mir::interpret::AllocBytes>::AllocParams {
+    }
 }
 
 // Please do not add any code below the above `Machine` trait impl. I (oli-obk) plan more cleanups
diff --git a/compiler/rustc_const_eval/src/const_eval/mod.rs b/compiler/rustc_const_eval/src/const_eval/mod.rs
index c0438fb3ff8..6fd0b9d26e3 100644
--- a/compiler/rustc_const_eval/src/const_eval/mod.rs
+++ b/compiler/rustc_const_eval/src/const_eval/mod.rs
@@ -1,6 +1,6 @@
 // Not in interpret to make sure we do not use private implementation details
 
-use rustc_abi::VariantIdx;
+use rustc_abi::{FieldIdx, VariantIdx};
 use rustc_middle::query::Key;
 use rustc_middle::ty::layout::LayoutOf;
 use rustc_middle::ty::{self, Ty, TyCtxt};
@@ -60,7 +60,7 @@ pub(crate) fn try_destructure_mir_constant_for_user_output<'tcx>(
 
     let fields_iter = (0..field_count)
         .map(|i| {
-            let field_op = ecx.project_field(&down, i).discard_err()?;
+            let field_op = ecx.project_field(&down, FieldIdx::from_usize(i)).discard_err()?;
             let val = op_to_const(&ecx, &field_op, /* for diagnostics */ true);
             Some((val, field_op.layout.ty))
         })
diff --git a/compiler/rustc_const_eval/src/const_eval/valtrees.rs b/compiler/rustc_const_eval/src/const_eval/valtrees.rs
index 34239ae1d15..58d230af683 100644
--- a/compiler/rustc_const_eval/src/const_eval/valtrees.rs
+++ b/compiler/rustc_const_eval/src/const_eval/valtrees.rs
@@ -1,4 +1,4 @@
-use rustc_abi::{BackendRepr, VariantIdx};
+use rustc_abi::{BackendRepr, FieldIdx, VariantIdx};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_middle::mir::interpret::{EvalToValTreeResult, GlobalId, ReportedErrorInfo};
 use rustc_middle::ty::layout::{LayoutCx, LayoutOf, TyAndLayout};
@@ -40,7 +40,7 @@ fn branches<'tcx>(
     }
 
     for i in 0..field_count {
-        let field = ecx.project_field(&place, i).unwrap();
+        let field = ecx.project_field(&place, FieldIdx::from_usize(i)).unwrap();
         let valtree = const_to_valtree_inner(ecx, &field, num_nodes)?;
         branches.push(valtree);
     }
@@ -437,7 +437,7 @@ fn valtree_into_mplace<'tcx>(
                     ty::Str | ty::Slice(_) | ty::Array(..) => {
                         ecx.project_index(place, i as u64).unwrap()
                     }
-                    _ => ecx.project_field(&place_adjusted, i).unwrap(),
+                    _ => ecx.project_field(&place_adjusted, FieldIdx::from_usize(i)).unwrap(),
                 };
 
                 debug!(?place_inner);
diff --git a/compiler/rustc_const_eval/src/errors.rs b/compiler/rustc_const_eval/src/errors.rs
index 826ea0e58ec..037cbf777e7 100644
--- a/compiler/rustc_const_eval/src/errors.rs
+++ b/compiler/rustc_const_eval/src/errors.rs
@@ -58,11 +58,6 @@ pub(crate) struct UnstableInStableExposed {
         code = "#[rustc_const_unstable(feature = \"...\", issue = \"...\")]\n",
         applicability = "has-placeholders"
     )]
-    #[suggestion(
-        const_eval_bypass_sugg,
-        code = "#[rustc_allow_const_fn_unstable({gate})]\n",
-        applicability = "has-placeholders"
-    )]
     pub attr_span: Span,
 }
 
@@ -444,38 +439,6 @@ pub struct LiveDrop<'tcx> {
     pub dropped_at: Span,
 }
 
-#[derive(Diagnostic)]
-#[diag(const_eval_error, code = E0080)]
-pub struct ConstEvalError {
-    #[primary_span]
-    pub span: Span,
-    /// One of "const", "const_with_path", and "static"
-    pub error_kind: &'static str,
-    pub instance: String,
-    #[subdiagnostic]
-    pub frame_notes: Vec<FrameNote>,
-}
-
-#[derive(Diagnostic)]
-#[diag(const_eval_nullary_intrinsic_fail)]
-pub struct NullaryIntrinsicError {
-    #[primary_span]
-    pub span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(const_eval_validation_failure, code = E0080)]
-pub struct ValidationFailure {
-    #[primary_span]
-    pub span: Span,
-    #[note(const_eval_validation_failure_note)]
-    pub ub_note: (),
-    #[subdiagnostic]
-    pub frames: Vec<FrameNote>,
-    #[subdiagnostic]
-    pub raw_bytes: RawBytesNote,
-}
-
 pub trait ReportErrorExt {
     /// Returns the diagnostic message for this error.
     fn diagnostic_message(&self) -> DiagMessage;
diff --git a/compiler/rustc_const_eval/src/interpret/call.rs b/compiler/rustc_const_eval/src/interpret/call.rs
index 405208e94f4..37677f9e048 100644
--- a/compiler/rustc_const_eval/src/interpret/call.rs
+++ b/compiler/rustc_const_eval/src/interpret/call.rs
@@ -62,7 +62,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     pub(super) fn fn_arg_field(
         &self,
         arg: &FnArg<'tcx, M::Provenance>,
-        field: usize,
+        field: FieldIdx,
     ) -> InterpResult<'tcx, FnArg<'tcx, M::Provenance>> {
         interp_ok(match arg {
             FnArg::Copy(op) => FnArg::Copy(self.project_field(op, field)?),
@@ -339,7 +339,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         caller_fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
         args: &[FnArg<'tcx, M::Provenance>],
         with_caller_location: bool,
-        destination: &MPlaceTy<'tcx, M::Provenance>,
+        destination: &PlaceTy<'tcx, M::Provenance>,
         mut stack_pop: StackPopCleanup,
     ) -> InterpResult<'tcx> {
         // Compute callee information.
@@ -487,7 +487,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             }
 
             // Protect return place for in-place return value passing.
-            M::protect_in_place_function_argument(self, &destination)?;
+            // We only need to protect anything if this is actually an in-memory place.
+            if let Left(mplace) = destination.as_mplace_or_local() {
+                M::protect_in_place_function_argument(self, &mplace)?;
+            }
 
             // Don't forget to mark "initially live" locals as live.
             self.storage_live_for_always_live_locals()?;
@@ -512,7 +515,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         (caller_abi, caller_fn_abi): (ExternAbi, &FnAbi<'tcx, Ty<'tcx>>),
         args: &[FnArg<'tcx, M::Provenance>],
         with_caller_location: bool,
-        destination: &MPlaceTy<'tcx, M::Provenance>,
+        destination: &PlaceTy<'tcx, M::Provenance>,
         target: Option<mir::BasicBlock>,
         unwind: mir::UnwindAction,
     ) -> InterpResult<'tcx> {
@@ -597,10 +600,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                         Cow::from(
                             args.iter()
                                 .map(|a| interp_ok(a.clone()))
-                                .chain(
-                                    (0..untuple_arg.layout().fields.count())
-                                        .map(|i| self.fn_arg_field(untuple_arg, i)),
-                                )
+                                .chain((0..untuple_arg.layout().fields.count()).map(|i| {
+                                    self.fn_arg_field(untuple_arg, FieldIdx::from_usize(i))
+                                }))
                                 .collect::<InterpResult<'_, Vec<_>>>()?,
                         )
                     } else {
@@ -776,10 +778,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         // Note that we are using `pop_stack_frame_raw` and not `return_from_current_stack_frame`,
         // as the latter "executes" the goto to the return block, but we don't want to,
         // only the tail called function should return to the current return block.
-        M::before_stack_pop(self, self.frame())?;
-
-        let StackPopInfo { return_action, return_to_block, return_place } =
-            self.pop_stack_frame_raw(false)?;
+        let StackPopInfo { return_action, return_to_block, return_place } = self
+            .pop_stack_frame_raw(false, |_this, _return_place| {
+                // This function's return value is just discarded, the tail-callee will fill in the return place instead.
+                interp_ok(())
+            })?;
 
         assert_eq!(return_action, ReturnAction::Normal);
 
@@ -850,7 +853,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             (ExternAbi::Rust, fn_abi),
             &[FnArg::Copy(arg.into())],
             false,
-            &ret,
+            &ret.into(),
             Some(target),
             unwind,
         )
@@ -891,28 +894,16 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             throw_ub_custom!(fluent::const_eval_unwind_past_top);
         }
 
-        M::before_stack_pop(self, self.frame())?;
-
-        // Copy return value. Must of course happen *before* we deallocate the locals.
-        // Must be *after* `before_stack_pop` as otherwise the return place might still be protected.
-        let copy_ret_result = if !unwinding {
-            let op = self
-                .local_to_op(mir::RETURN_PLACE, None)
-                .expect("return place should always be live");
-            let dest = self.frame().return_place.clone();
-            let res = self.copy_op_allow_transmute(&op, &dest);
-            trace!("return value: {:?}", self.dump_place(&dest.into()));
-            // We delay actually short-circuiting on this error until *after* the stack frame is
-            // popped, since we want this error to be attributed to the caller, whose type defines
-            // this transmute.
-            res
-        } else {
+        // Get out the return value. Must happen *before* the frame is popped as we have to get the
+        // local's value out.
+        let return_op =
+            self.local_to_op(mir::RETURN_PLACE, None).expect("return place should always be live");
+        // Do the actual pop + copy.
+        let stack_pop_info = self.pop_stack_frame_raw(unwinding, |this, return_place| {
+            this.copy_op_allow_transmute(&return_op, return_place)?;
+            trace!("return value: {:?}", this.dump_place(return_place));
             interp_ok(())
-        };
-
-        // All right, now it is time to actually pop the frame.
-        // An error here takes precedence over the copy error.
-        let (stack_pop_info, ()) = self.pop_stack_frame_raw(unwinding).and(copy_ret_result)?;
+        })?;
 
         match stack_pop_info.return_action {
             ReturnAction::Normal => {}
@@ -924,7 +915,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 // If we are not doing cleanup, also skip everything else.
                 assert!(self.stack().is_empty(), "only the topmost frame should ever be leaked");
                 assert!(!unwinding, "tried to skip cleanup during unwinding");
-                // Skip machine hook.
+                // Don't jump anywhere.
                 return interp_ok(());
             }
         }
diff --git a/compiler/rustc_const_eval/src/interpret/cast.rs b/compiler/rustc_const_eval/src/interpret/cast.rs
index 643a5805019..9e15f4572d7 100644
--- a/compiler/rustc_const_eval/src/interpret/cast.rs
+++ b/compiler/rustc_const_eval/src/interpret/cast.rs
@@ -1,6 +1,6 @@
 use std::assert_matches::assert_matches;
 
-use rustc_abi::Integer;
+use rustc_abi::{FieldIdx, Integer};
 use rustc_apfloat::ieee::{Double, Half, Quad, Single};
 use rustc_apfloat::{Float, FloatConvert};
 use rustc_middle::mir::CastKind;
@@ -484,6 +484,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 let mut found_cast_field = false;
                 for i in 0..src.layout.fields.count() {
                     let cast_ty_field = cast_ty.field(self, i);
+                    let i = FieldIdx::from_usize(i);
                     let src_field = self.project_field(src, i)?;
                     let dst_field = self.project_field(dest, i)?;
                     if src_field.layout.is_1zst() && cast_ty_field.is_1zst() {
diff --git a/compiler/rustc_const_eval/src/interpret/discriminant.rs b/compiler/rustc_const_eval/src/interpret/discriminant.rs
index 2f0b1cb6d1e..6c4b000e16b 100644
--- a/compiler/rustc_const_eval/src/interpret/discriminant.rs
+++ b/compiler/rustc_const_eval/src/interpret/discriminant.rs
@@ -1,6 +1,6 @@
 //! Functions for reading and writing discriminants of multi-variant layouts (enums and coroutines).
 
-use rustc_abi::{self as abi, TagEncoding, VariantIdx, Variants};
+use rustc_abi::{self as abi, FieldIdx, TagEncoding, VariantIdx, Variants};
 use rustc_middle::ty::layout::{LayoutOf, PrimitiveExt, TyAndLayout};
 use rustc_middle::ty::{self, CoroutineArgsExt, ScalarInt, Ty};
 use rustc_middle::{mir, span_bug};
@@ -231,7 +231,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         &self,
         layout: TyAndLayout<'tcx>,
         variant_index: VariantIdx,
-    ) -> InterpResult<'tcx, Option<(ScalarInt, usize)>> {
+    ) -> InterpResult<'tcx, Option<(ScalarInt, FieldIdx)>> {
         // Layout computation excludes uninhabited variants from consideration.
         // Therefore, there's no way to represent those variants in the given layout.
         // Essentially, uninhabited variants do not have a tag that corresponds to their
diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
index 090b2a692cf..ab27182c211 100644
--- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs
+++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
@@ -17,16 +17,16 @@ use tracing::trace;
 use super::memory::MemoryKind;
 use super::util::ensure_monomorphic_enough;
 use super::{
-    Allocation, CheckInAllocMsg, ConstAllocation, GlobalId, ImmTy, InterpCx, InterpResult,
-    MPlaceTy, Machine, OpTy, Pointer, PointerArithmetic, Provenance, Scalar, err_inval,
-    err_ub_custom, err_unsup_format, interp_ok, throw_inval, throw_ub_custom, throw_ub_format,
+    Allocation, CheckInAllocMsg, ConstAllocation, GlobalId, ImmTy, InterpCx, InterpResult, Machine,
+    OpTy, PlaceTy, Pointer, PointerArithmetic, Provenance, Scalar, err_inval, err_ub_custom,
+    err_unsup_format, interp_ok, throw_inval, throw_ub_custom, throw_ub_format,
 };
 use crate::fluent_generated as fluent;
 
 /// Directly returns an `Allocation` containing an absolute path representation of the given type.
 pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ConstAllocation<'tcx> {
     let path = crate::util::type_name(tcx, ty);
-    let alloc = Allocation::from_bytes_byte_aligned_immutable(path.into_bytes());
+    let alloc = Allocation::from_bytes_byte_aligned_immutable(path.into_bytes(), ());
     tcx.mk_const_alloc(alloc)
 }
 
@@ -50,13 +50,6 @@ pub(crate) fn eval_nullary_intrinsic<'tcx>(
             ensure_monomorphic_enough(tcx, tp_ty)?;
             ConstValue::from_bool(tp_ty.needs_drop(tcx, typing_env))
         }
-        sym::pref_align_of => {
-            // Correctly handles non-monomorphic calls, so there is no need for ensure_monomorphic_enough.
-            let layout = tcx
-                .layout_of(typing_env.as_query_input(tp_ty))
-                .map_err(|e| err_inval!(Layout(*e)))?;
-            ConstValue::from_target_usize(layout.align.pref.bytes(), &tcx)
-        }
         sym::type_id => {
             ensure_monomorphic_enough(tcx, tp_ty)?;
             ConstValue::from_u128(tcx.type_id_hash(tp_ty).as_u128())
@@ -112,7 +105,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         &mut self,
         instance: ty::Instance<'tcx>,
         args: &[OpTy<'tcx, M::Provenance>],
-        dest: &MPlaceTy<'tcx, M::Provenance>,
+        dest: &PlaceTy<'tcx, M::Provenance>,
         ret: Option<mir::BasicBlock>,
     ) -> InterpResult<'tcx, bool> {
         let instance_args = instance.args;
@@ -144,14 +137,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 self.write_scalar(Scalar::from_target_usize(result, self), dest)?;
             }
 
-            sym::pref_align_of
-            | sym::needs_drop
-            | sym::type_id
-            | sym::type_name
-            | sym::variant_count => {
+            sym::needs_drop | sym::type_id | sym::type_name | sym::variant_count => {
                 let gid = GlobalId { instance, promoted: None };
                 let ty = match intrinsic_name {
-                    sym::pref_align_of | sym::variant_count => self.tcx.types.usize,
+                    sym::variant_count => self.tcx.types.usize,
                     sym::needs_drop => self.tcx.types.bool,
                     sym::type_id => self.tcx.types.u128,
                     sym::type_name => Ty::new_static_str(self.tcx.tcx),
@@ -518,6 +507,103 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             sym::fabsf64 => self.float_abs_intrinsic::<Double>(args, dest)?,
             sym::fabsf128 => self.float_abs_intrinsic::<Quad>(args, dest)?,
 
+            sym::floorf16 => self.float_round_intrinsic::<Half>(
+                args,
+                dest,
+                rustc_apfloat::Round::TowardNegative,
+            )?,
+            sym::floorf32 => self.float_round_intrinsic::<Single>(
+                args,
+                dest,
+                rustc_apfloat::Round::TowardNegative,
+            )?,
+            sym::floorf64 => self.float_round_intrinsic::<Double>(
+                args,
+                dest,
+                rustc_apfloat::Round::TowardNegative,
+            )?,
+            sym::floorf128 => self.float_round_intrinsic::<Quad>(
+                args,
+                dest,
+                rustc_apfloat::Round::TowardNegative,
+            )?,
+
+            sym::ceilf16 => self.float_round_intrinsic::<Half>(
+                args,
+                dest,
+                rustc_apfloat::Round::TowardPositive,
+            )?,
+            sym::ceilf32 => self.float_round_intrinsic::<Single>(
+                args,
+                dest,
+                rustc_apfloat::Round::TowardPositive,
+            )?,
+            sym::ceilf64 => self.float_round_intrinsic::<Double>(
+                args,
+                dest,
+                rustc_apfloat::Round::TowardPositive,
+            )?,
+            sym::ceilf128 => self.float_round_intrinsic::<Quad>(
+                args,
+                dest,
+                rustc_apfloat::Round::TowardPositive,
+            )?,
+
+            sym::truncf16 => {
+                self.float_round_intrinsic::<Half>(args, dest, rustc_apfloat::Round::TowardZero)?
+            }
+            sym::truncf32 => {
+                self.float_round_intrinsic::<Single>(args, dest, rustc_apfloat::Round::TowardZero)?
+            }
+            sym::truncf64 => {
+                self.float_round_intrinsic::<Double>(args, dest, rustc_apfloat::Round::TowardZero)?
+            }
+            sym::truncf128 => {
+                self.float_round_intrinsic::<Quad>(args, dest, rustc_apfloat::Round::TowardZero)?
+            }
+
+            sym::roundf16 => self.float_round_intrinsic::<Half>(
+                args,
+                dest,
+                rustc_apfloat::Round::NearestTiesToAway,
+            )?,
+            sym::roundf32 => self.float_round_intrinsic::<Single>(
+                args,
+                dest,
+                rustc_apfloat::Round::NearestTiesToAway,
+            )?,
+            sym::roundf64 => self.float_round_intrinsic::<Double>(
+                args,
+                dest,
+                rustc_apfloat::Round::NearestTiesToAway,
+            )?,
+            sym::roundf128 => self.float_round_intrinsic::<Quad>(
+                args,
+                dest,
+                rustc_apfloat::Round::NearestTiesToAway,
+            )?,
+
+            sym::round_ties_even_f16 => self.float_round_intrinsic::<Half>(
+                args,
+                dest,
+                rustc_apfloat::Round::NearestTiesToEven,
+            )?,
+            sym::round_ties_even_f32 => self.float_round_intrinsic::<Single>(
+                args,
+                dest,
+                rustc_apfloat::Round::NearestTiesToEven,
+            )?,
+            sym::round_ties_even_f64 => self.float_round_intrinsic::<Double>(
+                args,
+                dest,
+                rustc_apfloat::Round::NearestTiesToEven,
+            )?,
+            sym::round_ties_even_f128 => self.float_round_intrinsic::<Quad>(
+                args,
+                dest,
+                rustc_apfloat::Round::NearestTiesToEven,
+            )?,
+
             // Unsupported intrinsic: skip the return_to_block below.
             _ => return interp_ok(false),
         }
@@ -587,7 +673,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         &mut self,
         a: &ImmTy<'tcx, M::Provenance>,
         b: &ImmTy<'tcx, M::Provenance>,
-        dest: &MPlaceTy<'tcx, M::Provenance>,
+        dest: &PlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx> {
         assert_eq!(a.layout.ty, b.layout.ty);
         assert_matches!(a.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
@@ -801,7 +887,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     fn float_min_intrinsic<F>(
         &mut self,
         args: &[OpTy<'tcx, M::Provenance>],
-        dest: &MPlaceTy<'tcx, M::Provenance>,
+        dest: &PlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx, ()>
     where
         F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
@@ -822,7 +908,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     fn float_max_intrinsic<F>(
         &mut self,
         args: &[OpTy<'tcx, M::Provenance>],
-        dest: &MPlaceTy<'tcx, M::Provenance>,
+        dest: &PlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx, ()>
     where
         F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
@@ -843,7 +929,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     fn float_minimum_intrinsic<F>(
         &mut self,
         args: &[OpTy<'tcx, M::Provenance>],
-        dest: &MPlaceTy<'tcx, M::Provenance>,
+        dest: &PlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx, ()>
     where
         F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
@@ -859,7 +945,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     fn float_maximum_intrinsic<F>(
         &mut self,
         args: &[OpTy<'tcx, M::Provenance>],
-        dest: &MPlaceTy<'tcx, M::Provenance>,
+        dest: &PlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx, ()>
     where
         F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
@@ -875,7 +961,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     fn float_copysign_intrinsic<F>(
         &mut self,
         args: &[OpTy<'tcx, M::Provenance>],
-        dest: &MPlaceTy<'tcx, M::Provenance>,
+        dest: &PlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx, ()>
     where
         F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
@@ -890,7 +976,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     fn float_abs_intrinsic<F>(
         &mut self,
         args: &[OpTy<'tcx, M::Provenance>],
-        dest: &MPlaceTy<'tcx, M::Provenance>,
+        dest: &PlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx, ()>
     where
         F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
@@ -900,4 +986,20 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         self.write_scalar(x.abs(), dest)?;
         interp_ok(())
     }
+
+    fn float_round_intrinsic<F>(
+        &mut self,
+        args: &[OpTy<'tcx, M::Provenance>],
+        dest: &PlaceTy<'tcx, M::Provenance>,
+        mode: rustc_apfloat::Round,
+    ) -> InterpResult<'tcx, ()>
+    where
+        F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
+    {
+        let x: F = self.read_scalar(&args[0])?.to_float()?;
+        let res = x.round_to_integral(mode).value;
+        let res = self.adjust_nan(res, &[x]);
+        self.write_scalar(res, dest)?;
+        interp_ok(())
+    }
 }
diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs
index a1386b4e1be..b9e022c9604 100644
--- a/compiler/rustc_const_eval/src/interpret/machine.rs
+++ b/compiler/rustc_const_eval/src/interpret/machine.rs
@@ -147,6 +147,12 @@ pub trait Machine<'tcx>: Sized {
     /// already been checked before.
     const ALL_CONSTS_ARE_PRECHECKED: bool = true;
 
+    /// Determines whether rustc_const_eval functions that make use of the [Machine] should make
+    /// tracing calls (to the `tracing` library). By default this is `false`, meaning the tracing
+    /// calls will supposedly be optimized out. This flag is set to `true` inside Miri, to allow
+    /// tracing the interpretation steps, among other things.
+    const TRACING_ENABLED: bool = false;
+
     /// Whether memory accesses should be alignment-checked.
     fn enforce_alignment(ecx: &InterpCx<'tcx, Self>) -> bool;
 
@@ -202,7 +208,7 @@ pub trait Machine<'tcx>: Sized {
         instance: ty::Instance<'tcx>,
         abi: &FnAbi<'tcx, Ty<'tcx>>,
         args: &[FnArg<'tcx, Self::Provenance>],
-        destination: &MPlaceTy<'tcx, Self::Provenance>,
+        destination: &PlaceTy<'tcx, Self::Provenance>,
         target: Option<mir::BasicBlock>,
         unwind: mir::UnwindAction,
     ) -> InterpResult<'tcx, Option<(&'tcx mir::Body<'tcx>, ty::Instance<'tcx>)>>;
@@ -214,7 +220,7 @@ pub trait Machine<'tcx>: Sized {
         fn_val: Self::ExtraFnVal,
         abi: &FnAbi<'tcx, Ty<'tcx>>,
         args: &[FnArg<'tcx, Self::Provenance>],
-        destination: &MPlaceTy<'tcx, Self::Provenance>,
+        destination: &PlaceTy<'tcx, Self::Provenance>,
         target: Option<mir::BasicBlock>,
         unwind: mir::UnwindAction,
     ) -> InterpResult<'tcx>;
@@ -228,7 +234,7 @@ pub trait Machine<'tcx>: Sized {
         ecx: &mut InterpCx<'tcx, Self>,
         instance: ty::Instance<'tcx>,
         args: &[OpTy<'tcx, Self::Provenance>],
-        destination: &MPlaceTy<'tcx, Self::Provenance>,
+        destination: &PlaceTy<'tcx, Self::Provenance>,
         target: Option<mir::BasicBlock>,
         unwind: mir::UnwindAction,
     ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>>;
@@ -530,11 +536,9 @@ pub trait Machine<'tcx>: Sized {
         interp_ok(())
     }
 
-    /// Called just before the return value is copied to the caller-provided return place.
-    fn before_stack_pop(
-        _ecx: &InterpCx<'tcx, Self>,
-        _frame: &Frame<'tcx, Self::Provenance, Self::FrameExtra>,
-    ) -> InterpResult<'tcx> {
+    /// Called just before the frame is removed from the stack (followed by return value copy and
+    /// local cleanup).
+    fn before_stack_pop(_ecx: &mut InterpCx<'tcx, Self>) -> InterpResult<'tcx> {
         interp_ok(())
     }
 
@@ -622,6 +626,10 @@ pub trait Machine<'tcx>: Sized {
         // Default to no caching.
         Cow::Owned(compute_range())
     }
+
+    /// Compute the value passed to the constructors of the `AllocBytes` type for
+    /// abstract machine allocations.
+    fn get_default_alloc_params(&self) -> <Self::Bytes as AllocBytes>::AllocParams;
 }
 
 /// A lot of the flexibility above is just needed for `Miri`, but all "compile-time" machines
@@ -669,7 +677,7 @@ pub macro compile_time_machine(<$tcx: lifetime>) {
         fn_val: !,
         _abi: &FnAbi<$tcx, Ty<$tcx>>,
         _args: &[FnArg<$tcx>],
-        _destination: &MPlaceTy<$tcx, Self::Provenance>,
+        _destination: &PlaceTy<$tcx, Self::Provenance>,
         _target: Option<mir::BasicBlock>,
         _unwind: mir::UnwindAction,
     ) -> InterpResult<$tcx> {
diff --git a/compiler/rustc_const_eval/src/interpret/memory.rs b/compiler/rustc_const_eval/src/interpret/memory.rs
index 43bf48a9b96..99a4bc1b7d6 100644
--- a/compiler/rustc_const_eval/src/interpret/memory.rs
+++ b/compiler/rustc_const_eval/src/interpret/memory.rs
@@ -233,10 +233,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         kind: MemoryKind<M::MemoryKind>,
         init: AllocInit,
     ) -> InterpResult<'tcx, Pointer<M::Provenance>> {
+        let params = self.machine.get_default_alloc_params();
         let alloc = if M::PANIC_ON_ALLOC_FAIL {
-            Allocation::new(size, align, init)
+            Allocation::new(size, align, init, params)
         } else {
-            Allocation::try_new(size, align, init)?
+            Allocation::try_new(size, align, init, params)?
         };
         self.insert_allocation(alloc, kind)
     }
@@ -248,7 +249,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         kind: MemoryKind<M::MemoryKind>,
         mutability: Mutability,
     ) -> InterpResult<'tcx, Pointer<M::Provenance>> {
-        let alloc = Allocation::from_bytes(bytes, align, mutability);
+        let params = self.machine.get_default_alloc_params();
+        let alloc = Allocation::from_bytes(bytes, align, mutability, params);
         self.insert_allocation(alloc, kind)
     }
 
diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs
index 36da9037e43..77667ba823a 100644
--- a/compiler/rustc_const_eval/src/interpret/operand.rs
+++ b/compiler/rustc_const_eval/src/interpret/operand.rs
@@ -12,6 +12,7 @@ use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, LayoutOf, TyAndLayout};
 use rustc_middle::ty::print::{FmtPrinter, PrettyPrinter};
 use rustc_middle::ty::{ConstInt, ScalarInt, Ty, TyCtxt};
 use rustc_middle::{bug, mir, span_bug, ty};
+use rustc_span::DUMMY_SP;
 use tracing::trace;
 
 use super::{
@@ -307,10 +308,10 @@ impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> {
     #[inline]
     pub fn from_ordering(c: std::cmp::Ordering, tcx: TyCtxt<'tcx>) -> Self {
         // Can use any typing env, since `Ordering` is always monomorphic.
-        let ty = tcx.ty_ordering_enum(None);
+        let ty = tcx.ty_ordering_enum(DUMMY_SP);
         let layout =
             tcx.layout_of(ty::TypingEnv::fully_monomorphized().as_query_input(ty)).unwrap();
-        Self::from_scalar(Scalar::from_i8(c as i8), layout)
+        Self::from_scalar(Scalar::Int(c.into()), layout)
     }
 
     pub fn from_pair(a: Self, b: Self, cx: &(impl HasTypingEnv<'tcx> + HasTyCtxt<'tcx>)) -> Self {
diff --git a/compiler/rustc_const_eval/src/interpret/projection.rs b/compiler/rustc_const_eval/src/interpret/projection.rs
index 8ecb3e13d5c..ad47a19a14d 100644
--- a/compiler/rustc_const_eval/src/interpret/projection.rs
+++ b/compiler/rustc_const_eval/src/interpret/projection.rs
@@ -10,7 +10,7 @@
 use std::marker::PhantomData;
 use std::ops::Range;
 
-use rustc_abi::{self as abi, Size, VariantIdx};
+use rustc_abi::{self as abi, FieldIdx, Size, VariantIdx};
 use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
 use rustc_middle::{bug, mir, span_bug, ty};
@@ -144,22 +144,22 @@ where
     /// always possible without allocating, so it can take `&self`. Also return the field's layout.
     /// This supports both struct and array fields, but not slices!
     ///
-    /// This also works for arrays, but then the `usize` index type is restricting.
-    /// For indexing into arrays, use `mplace_index`.
+    /// This also works for arrays, but then the `FieldIdx` index type is restricting.
+    /// For indexing into arrays, use [`Self::project_index`].
     pub fn project_field<P: Projectable<'tcx, M::Provenance>>(
         &self,
         base: &P,
-        field: usize,
+        field: FieldIdx,
     ) -> InterpResult<'tcx, P> {
         // Slices nominally have length 0, so they will panic somewhere in `fields.offset`.
         debug_assert!(
             !matches!(base.layout().ty.kind(), ty::Slice(..)),
             "`field` projection called on a slice -- call `index` projection instead"
         );
-        let offset = base.layout().fields.offset(field);
+        let offset = base.layout().fields.offset(field.as_usize());
         // Computing the layout does normalization, so we get a normalized type out of this
         // even if the field type is non-normalized (possible e.g. via associated types).
-        let field_layout = base.layout().field(self, field);
+        let field_layout = base.layout().field(self, field.as_usize());
 
         // Offset may need adjustment for unsized fields.
         let (meta, offset) = if field_layout.is_unsized() {
@@ -244,7 +244,7 @@ where
             }
             _ => span_bug!(
                 self.cur_span(),
-                "`mplace_index` called on non-array type {:?}",
+                "`project_index` called on non-array type {:?}",
                 base.layout().ty
             ),
         };
@@ -260,7 +260,7 @@ where
     ) -> InterpResult<'tcx, (P, u64)> {
         assert!(base.layout().ty.ty_adt_def().unwrap().repr().simd());
         // SIMD types must be newtypes around arrays, so all we have to do is project to their only field.
-        let array = self.project_field(base, 0)?;
+        let array = self.project_field(base, FieldIdx::ZERO)?;
         let len = array.len(self)?;
         interp_ok((array, len))
     }
@@ -384,7 +384,7 @@ where
             UnwrapUnsafeBinder(target) => base.transmute(self.layout_of(target)?, self)?,
             // We don't want anything happening here, this is here as a dummy.
             Subtype(_) => base.transmute(base.layout(), self)?,
-            Field(field, _) => self.project_field(base, field.index())?,
+            Field(field, _) => self.project_field(base, field)?,
             Downcast(_, variant) => self.project_downcast(base, variant)?,
             Deref => self.deref_pointer(&base.to_op(self)?)?.into(),
             Index(local) => {
diff --git a/compiler/rustc_const_eval/src/interpret/stack.rs b/compiler/rustc_const_eval/src/interpret/stack.rs
index d7b03776bc4..2a2d1bb2754 100644
--- a/compiler/rustc_const_eval/src/interpret/stack.rs
+++ b/compiler/rustc_const_eval/src/interpret/stack.rs
@@ -15,9 +15,9 @@ use rustc_span::Span;
 use tracing::{info_span, instrument, trace};
 
 use super::{
-    AllocId, CtfeProvenance, Immediate, InterpCx, InterpResult, MPlaceTy, Machine, MemPlace,
-    MemPlaceMeta, MemoryKind, Operand, Pointer, Provenance, ReturnAction, Scalar,
-    from_known_layout, interp_ok, throw_ub, throw_unsup,
+    AllocId, CtfeProvenance, Immediate, InterpCx, InterpResult, Machine, MemPlace, MemPlaceMeta,
+    MemoryKind, Operand, PlaceTy, Pointer, Provenance, ReturnAction, Scalar, from_known_layout,
+    interp_ok, throw_ub, throw_unsup,
 };
 use crate::errors;
 
@@ -76,8 +76,10 @@ pub struct Frame<'tcx, Prov: Provenance = CtfeProvenance, Extra = ()> {
     return_to_block: StackPopCleanup,
 
     /// The location where the result of the current stack frame should be written to,
-    /// and its layout in the caller.
-    pub return_place: MPlaceTy<'tcx, Prov>,
+    /// and its layout in the caller. This place is to be interpreted relative to the
+    /// *caller's* stack frame. We use a `PlaceTy` instead of an `MPlaceTy` since this
+    /// avoids having to move *all* return places into Miri's memory.
+    pub return_place: PlaceTy<'tcx, Prov>,
 
     /// The list of locals for this stack frame, stored in order as
     /// `[return_ptr, arguments..., variables..., temporaries...]`.
@@ -129,7 +131,7 @@ pub struct StackPopInfo<'tcx, Prov: Provenance> {
     pub return_to_block: StackPopCleanup,
 
     /// [`return_place`](Frame::return_place) of the popped stack frame.
-    pub return_place: MPlaceTy<'tcx, Prov>,
+    pub return_place: PlaceTy<'tcx, Prov>,
 }
 
 /// State of a local variable including a memoized layout
@@ -353,7 +355,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         &mut self,
         instance: ty::Instance<'tcx>,
         body: &'tcx mir::Body<'tcx>,
-        return_place: &MPlaceTy<'tcx, M::Provenance>,
+        return_place: &PlaceTy<'tcx, M::Provenance>,
         return_to_block: StackPopCleanup,
     ) -> InterpResult<'tcx> {
         trace!("body: {:#?}", body);
@@ -404,9 +406,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     /// it.
     ///
     /// This also deallocates locals, if necessary.
+    /// `copy_ret_val` gets called after the frame has been taken from the stack but before the locals have been deallocated.
     ///
-    /// [`M::before_stack_pop`] should be called before calling this function.
-    /// [`M::after_stack_pop`] is called by this function automatically.
+    /// [`M::before_stack_pop`] and [`M::after_stack_pop`] are called by this function
+    /// automatically.
     ///
     /// The high-level version of this is `return_from_current_stack_frame`.
     ///
@@ -415,47 +418,44 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     pub(super) fn pop_stack_frame_raw(
         &mut self,
         unwinding: bool,
+        copy_ret_val: impl FnOnce(&mut Self, &PlaceTy<'tcx, M::Provenance>) -> InterpResult<'tcx>,
     ) -> InterpResult<'tcx, StackPopInfo<'tcx, M::Provenance>> {
-        let cleanup = self.cleanup_current_frame_locals()?;
-
+        M::before_stack_pop(self)?;
         let frame =
             self.stack_mut().pop().expect("tried to pop a stack frame, but there were none");
 
+        // Copy return value (unless we are unwinding).
+        if !unwinding {
+            copy_ret_val(self, &frame.return_place)?;
+        }
+
         let return_to_block = frame.return_to_block;
         let return_place = frame.return_place.clone();
 
-        let return_action;
-        if cleanup {
-            return_action = M::after_stack_pop(self, frame, unwinding)?;
-            assert_ne!(return_action, ReturnAction::NoCleanup);
-        } else {
-            return_action = ReturnAction::NoCleanup;
-        };
-
-        interp_ok(StackPopInfo { return_action, return_to_block, return_place })
-    }
-
-    /// A private helper for [`pop_stack_frame_raw`](InterpCx::pop_stack_frame_raw).
-    /// Returns `true` if cleanup has been done, `false` otherwise.
-    fn cleanup_current_frame_locals(&mut self) -> InterpResult<'tcx, bool> {
         // Cleanup: deallocate locals.
         // Usually we want to clean up (deallocate locals), but in a few rare cases we don't.
         // We do this while the frame is still on the stack, so errors point to the callee.
-        let return_to_block = self.frame().return_to_block;
         let cleanup = match return_to_block {
             StackPopCleanup::Goto { .. } => true,
             StackPopCleanup::Root { cleanup, .. } => cleanup,
         };
 
-        if cleanup {
+        let return_action = if cleanup {
             // We need to take the locals out, since we need to mutate while iterating.
-            let locals = mem::take(&mut self.frame_mut().locals);
-            for local in &locals {
+            for local in &frame.locals {
                 self.deallocate_local(local.value)?;
             }
-        }
 
-        interp_ok(cleanup)
+            // Call the machine hook, which determines the next steps.
+            let return_action = M::after_stack_pop(self, frame, unwinding)?;
+            assert_ne!(return_action, ReturnAction::NoCleanup);
+            return_action
+        } else {
+            // We also skip the machine hook when there's no cleanup. This not a real "pop" anyway.
+            ReturnAction::NoCleanup
+        };
+
+        interp_ok(StackPopInfo { return_action, return_to_block, return_place })
     }
 
     /// In the current stack frame, mark all locals as live that are not arguments and don't have
diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs
index 363ceee1970..833fcc38817 100644
--- a/compiler/rustc_const_eval/src/interpret/step.rs
+++ b/compiler/rustc_const_eval/src/interpret/step.rs
@@ -333,7 +333,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         }
         for (field_index, operand) in operands.iter_enumerated() {
             let field_index = active_field_index.unwrap_or(field_index);
-            let field_dest = self.project_field(&variant_dest, field_index.as_usize())?;
+            let field_dest = self.project_field(&variant_dest, field_index)?;
             let op = self.eval_operand(operand, Some(field_dest.layout))?;
             self.copy_op(&op, &field_dest)?;
         }
@@ -506,7 +506,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } =
                     self.eval_callee_and_args(terminator, func, args)?;
 
-                let destination = self.force_allocation(&self.eval_place(destination)?)?;
+                let destination = self.eval_place(destination)?;
                 self.init_fn_call(
                     callee,
                     (fn_sig.abi, fn_abi),
diff --git a/compiler/rustc_const_eval/src/interpret/traits.rs b/compiler/rustc_const_eval/src/interpret/traits.rs
index a5029eea5a7..7249ef23bf6 100644
--- a/compiler/rustc_const_eval/src/interpret/traits.rs
+++ b/compiler/rustc_const_eval/src/interpret/traits.rs
@@ -1,4 +1,4 @@
-use rustc_abi::{Align, Size};
+use rustc_abi::{Align, FieldIdx, Size};
 use rustc_middle::mir::interpret::{InterpResult, Pointer};
 use rustc_middle::ty::layout::LayoutOf;
 use rustc_middle::ty::{self, ExistentialPredicateStableCmpExt, Ty, TyCtxt, VtblEntry};
@@ -137,8 +137,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             matches!(val.layout().ty.kind(), ty::Dynamic(_, _, ty::DynStar)),
             "`unpack_dyn_star` only makes sense on `dyn*` types"
         );
-        let data = self.project_field(val, 0)?;
-        let vtable = self.project_field(val, 1)?;
+        let data = self.project_field(val, FieldIdx::ZERO)?;
+        let vtable = self.project_field(val, FieldIdx::ONE)?;
         let vtable = self.read_pointer(&vtable.to_op(self)?)?;
         let ty = self.get_ptr_vtable_ty(vtable, Some(expected_trait))?;
         // `data` is already the right thing but has the wrong type. So we transmute it.
diff --git a/compiler/rustc_const_eval/src/interpret/util.rs b/compiler/rustc_const_eval/src/interpret/util.rs
index ba579e25f03..83a17092619 100644
--- a/compiler/rustc_const_eval/src/interpret/util.rs
+++ b/compiler/rustc_const_eval/src/interpret/util.rs
@@ -38,10 +38,29 @@ pub(crate) fn create_static_alloc<'tcx>(
     static_def_id: LocalDefId,
     layout: TyAndLayout<'tcx>,
 ) -> InterpResult<'tcx, MPlaceTy<'tcx>> {
-    let alloc = Allocation::try_new(layout.size, layout.align.abi, AllocInit::Uninit)?;
+    let alloc = Allocation::try_new(layout.size, layout.align.abi, AllocInit::Uninit, ())?;
     let alloc_id = ecx.tcx.reserve_and_set_static_alloc(static_def_id.into());
     assert_eq!(ecx.machine.static_root_ids, None);
     ecx.machine.static_root_ids = Some((alloc_id, static_def_id));
     assert!(ecx.memory.alloc_map.insert(alloc_id, (MemoryKind::Stack, alloc)).is_none());
     interp_ok(ecx.ptr_to_mplace(Pointer::from(alloc_id).into(), layout))
 }
+
+/// This struct is needed to enforce `#[must_use]` on [tracing::span::EnteredSpan]
+/// while wrapping them in an `Option`.
+#[must_use]
+pub enum MaybeEnteredSpan {
+    Some(tracing::span::EnteredSpan),
+    None,
+}
+
+#[macro_export]
+macro_rules! enter_trace_span {
+    ($machine:ident, $($tt:tt)*) => {
+        if $machine::TRACING_ENABLED {
+            $crate::interpret::tracing_utils::MaybeEnteredSpan::Some(tracing::info_span!($($tt)*).entered())
+        } else {
+            $crate::interpret::tracing_utils::MaybeEnteredSpan::None
+        }
+    }
+}
diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs
index 8f39afa642a..7d76d925ef2 100644
--- a/compiler/rustc_const_eval/src/interpret/validity.rs
+++ b/compiler/rustc_const_eval/src/interpret/validity.rs
@@ -294,7 +294,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
         // First, check if we are projecting to a variant.
         match layout.variants {
             Variants::Multiple { tag_field, .. } => {
-                if tag_field == field {
+                if tag_field.as_usize() == field {
                     return match layout.ty.kind() {
                         ty::Adt(def, ..) if def.is_enum() => PathElem::EnumTag,
                         ty::Coroutine(..) => PathElem::CoroutineTag,
diff --git a/compiler/rustc_const_eval/src/interpret/visitor.rs b/compiler/rustc_const_eval/src/interpret/visitor.rs
index 3647c109a6e..5aea91233bd 100644
--- a/compiler/rustc_const_eval/src/interpret/visitor.rs
+++ b/compiler/rustc_const_eval/src/interpret/visitor.rs
@@ -112,8 +112,10 @@ pub trait ValueVisitor<'tcx, M: Machine<'tcx>>: Sized {
                 // So we transmute it to a raw pointer.
                 let raw_ptr_ty = Ty::new_mut_ptr(*self.ecx().tcx, self.ecx().tcx.types.unit);
                 let raw_ptr_ty = self.ecx().layout_of(raw_ptr_ty)?;
-                let vtable_field =
-                    self.ecx().project_field(v, 1)?.transmute(raw_ptr_ty, self.ecx())?;
+                let vtable_field = self
+                    .ecx()
+                    .project_field(v, FieldIdx::ONE)?
+                    .transmute(raw_ptr_ty, self.ecx())?;
                 self.visit_field(v, 1, &vtable_field)?;
 
                 // Then unpack the first field, and continue.
@@ -140,14 +142,16 @@ pub trait ValueVisitor<'tcx, M: Machine<'tcx>>: Sized {
 
                 // `Box` has two fields: the pointer we care about, and the allocator.
                 assert_eq!(v.layout().fields.count(), 2, "`Box` must have exactly 2 fields");
-                let (unique_ptr, alloc) =
-                    (self.ecx().project_field(v, 0)?, self.ecx().project_field(v, 1)?);
+                let (unique_ptr, alloc) = (
+                    self.ecx().project_field(v, FieldIdx::ZERO)?,
+                    self.ecx().project_field(v, FieldIdx::ONE)?,
+                );
                 // Unfortunately there is some type junk in the way here: `unique_ptr` is a `Unique`...
                 // (which means another 2 fields, the second of which is a `PhantomData`)
                 assert_eq!(unique_ptr.layout().fields.count(), 2);
                 let (nonnull_ptr, phantom) = (
-                    self.ecx().project_field(&unique_ptr, 0)?,
-                    self.ecx().project_field(&unique_ptr, 1)?,
+                    self.ecx().project_field(&unique_ptr, FieldIdx::ZERO)?,
+                    self.ecx().project_field(&unique_ptr, FieldIdx::ONE)?,
                 );
                 assert!(
                     phantom.layout().ty.ty_adt_def().is_some_and(|adt| adt.is_phantom_data()),
@@ -156,7 +160,7 @@ pub trait ValueVisitor<'tcx, M: Machine<'tcx>>: Sized {
                 );
                 // ... that contains a `NonNull`... (gladly, only a single field here)
                 assert_eq!(nonnull_ptr.layout().fields.count(), 1);
-                let raw_ptr = self.ecx().project_field(&nonnull_ptr, 0)?; // the actual raw ptr
+                let raw_ptr = self.ecx().project_field(&nonnull_ptr, FieldIdx::ZERO)?; // the actual raw ptr
                 // ... whose only field finally is a raw ptr we can dereference.
                 self.visit_box(ty, &raw_ptr)?;
 
@@ -188,9 +192,8 @@ pub trait ValueVisitor<'tcx, M: Machine<'tcx>>: Sized {
             }
             FieldsShape::Arbitrary { memory_index, .. } => {
                 for idx in Self::aggregate_field_iter(memory_index) {
-                    let idx = idx.as_usize();
                     let field = self.ecx().project_field(v, idx)?;
-                    self.visit_field(v, idx, &field)?;
+                    self.visit_field(v, idx.as_usize(), &field)?;
                 }
             }
             FieldsShape::Array { .. } => {
diff --git a/compiler/rustc_const_eval/src/util/caller_location.rs b/compiler/rustc_const_eval/src/util/caller_location.rs
index e926040e9ba..671214002a0 100644
--- a/compiler/rustc_const_eval/src/util/caller_location.rs
+++ b/compiler/rustc_const_eval/src/util/caller_location.rs
@@ -1,3 +1,4 @@
+use rustc_abi::FieldIdx;
 use rustc_hir::LangItem;
 use rustc_middle::ty::layout::LayoutOf;
 use rustc_middle::ty::{self, TyCtxt};
@@ -15,32 +16,40 @@ fn alloc_caller_location<'tcx>(
     line: u32,
     col: u32,
 ) -> MPlaceTy<'tcx> {
+    // Ensure that the filename itself does not contain nul bytes.
+    // This isn't possible via POSIX or Windows, but we should ensure no one
+    // ever does such a thing.
+    assert!(!filename.as_str().as_bytes().contains(&0));
+
     let loc_details = ecx.tcx.sess.opts.unstable_opts.location_detail;
-    // This can fail if rustc runs out of memory right here. Trying to emit an error would be
-    // pointless, since that would require allocating more memory than these short strings.
-    let file = if loc_details.file {
-        ecx.allocate_str_dedup(filename.as_str()).unwrap()
-    } else {
-        ecx.allocate_str_dedup("<redacted>").unwrap()
+    let file_wide_ptr = {
+        let filename = if loc_details.file { filename.as_str() } else { "<redacted>" };
+        let filename_with_nul = filename.to_owned() + "\0";
+        // This can fail if rustc runs out of memory right here. Trying to emit an error would be
+        // pointless, since that would require allocating more memory than these short strings.
+        let file_ptr = ecx.allocate_bytes_dedup(filename_with_nul.as_bytes()).unwrap();
+        Immediate::new_slice(file_ptr.into(), filename_with_nul.len().try_into().unwrap(), ecx)
     };
-    let file = file.map_provenance(CtfeProvenance::as_immutable);
     let line = if loc_details.line { Scalar::from_u32(line) } else { Scalar::from_u32(0) };
     let col = if loc_details.column { Scalar::from_u32(col) } else { Scalar::from_u32(0) };
 
     // Allocate memory for `CallerLocation` struct.
     let loc_ty = ecx
         .tcx
-        .type_of(ecx.tcx.require_lang_item(LangItem::PanicLocation, None))
+        .type_of(ecx.tcx.require_lang_item(LangItem::PanicLocation, ecx.tcx.span))
         .instantiate(*ecx.tcx, ecx.tcx.mk_args(&[ecx.tcx.lifetimes.re_erased.into()]));
     let loc_layout = ecx.layout_of(loc_ty).unwrap();
     let location = ecx.allocate(loc_layout, MemoryKind::CallerLocation).unwrap();
 
     // Initialize fields.
-    ecx.write_immediate(file.to_ref(ecx), &ecx.project_field(&location, 0).unwrap())
-        .expect("writing to memory we just allocated cannot fail");
-    ecx.write_scalar(line, &ecx.project_field(&location, 1).unwrap())
+    ecx.write_immediate(
+        file_wide_ptr,
+        &ecx.project_field(&location, FieldIdx::from_u32(0)).unwrap(),
+    )
+    .expect("writing to memory we just allocated cannot fail");
+    ecx.write_scalar(line, &ecx.project_field(&location, FieldIdx::from_u32(1)).unwrap())
         .expect("writing to memory we just allocated cannot fail");
-    ecx.write_scalar(col, &ecx.project_field(&location, 2).unwrap())
+    ecx.write_scalar(col, &ecx.project_field(&location, FieldIdx::from_u32(2)).unwrap())
         .expect("writing to memory we just allocated cannot fail");
 
     location
diff --git a/compiler/rustc_const_eval/src/util/type_name.rs b/compiler/rustc_const_eval/src/util/type_name.rs
index 30e96ae4143..e8f2728a772 100644
--- a/compiler/rustc_const_eval/src/util/type_name.rs
+++ b/compiler/rustc_const_eval/src/util/type_name.rs
@@ -125,7 +125,7 @@ impl<'tcx> Printer<'tcx> for AbsolutePathPrinter<'tcx> {
     ) -> Result<(), PrintError> {
         print_prefix(self)?;
         let args =
-            args.iter().cloned().filter(|arg| !matches!(arg.unpack(), GenericArgKind::Lifetime(_)));
+            args.iter().cloned().filter(|arg| !matches!(arg.kind(), GenericArgKind::Lifetime(_)));
         if args.clone().next().is_some() {
             self.generic_delimiters(|cx| cx.comma_sep(args))
         } else {
diff --git a/compiler/rustc_data_structures/src/flock.rs b/compiler/rustc_data_structures/src/flock.rs
index d423d8acefd..60ae7ad115a 100644
--- a/compiler/rustc_data_structures/src/flock.rs
+++ b/compiler/rustc_data_structures/src/flock.rs
@@ -4,7 +4,18 @@
 //! green/native threading. This is just a bare-bones enough solution for
 //! librustdoc, it is not production quality at all.
 
-cfg_match! {
+// cfg(bootstrap)
+macro_rules! cfg_select_dispatch {
+    ($($tokens:tt)*) => {
+        #[cfg(bootstrap)]
+        cfg_match! { $($tokens)* }
+
+        #[cfg(not(bootstrap))]
+        cfg_select! { $($tokens)* }
+    };
+}
+
+cfg_select_dispatch! {
     target_os = "linux" => {
         mod linux;
         use linux as imp;
diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs
index 865424fd6bb..eb3817a80a7 100644
--- a/compiler/rustc_data_structures/src/lib.rs
+++ b/compiler/rustc_data_structures/src/lib.rs
@@ -10,6 +10,8 @@
 #![allow(internal_features)]
 #![allow(rustc::default_hash_types)]
 #![allow(rustc::potential_query_instability)]
+#![cfg_attr(bootstrap, feature(cfg_match))]
+#![cfg_attr(not(bootstrap), feature(cfg_select))]
 #![deny(unsafe_op_in_unsafe_fn)]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
@@ -19,12 +21,10 @@
 #![feature(ascii_char_variants)]
 #![feature(assert_matches)]
 #![feature(auto_traits)]
-#![feature(cfg_match)]
 #![feature(core_intrinsics)]
 #![feature(dropck_eyepatch)]
 #![feature(extend_one)]
 #![feature(file_buffered)]
-#![feature(macro_metavar_expr)]
 #![feature(map_try_insert)]
 #![feature(min_specialization)]
 #![feature(negative_impls)]
diff --git a/compiler/rustc_data_structures/src/profiling.rs b/compiler/rustc_data_structures/src/profiling.rs
index 60f007083ba..e3a01e4035c 100644
--- a/compiler/rustc_data_structures/src/profiling.rs
+++ b/compiler/rustc_data_structures/src/profiling.rs
@@ -859,8 +859,19 @@ fn get_thread_id() -> u32 {
     std::thread::current().id().as_u64().get() as u32
 }
 
+// cfg(bootstrap)
+macro_rules! cfg_select_dispatch {
+    ($($tokens:tt)*) => {
+        #[cfg(bootstrap)]
+        cfg_match! { $($tokens)* }
+
+        #[cfg(not(bootstrap))]
+        cfg_select! { $($tokens)* }
+    };
+}
+
 // Memory reporting
-cfg_match! {
+cfg_select_dispatch! {
     windows => {
         pub fn get_resident_set_size() -> Option<usize> {
             use windows::{
diff --git a/compiler/rustc_data_structures/src/transitive_relation.rs b/compiler/rustc_data_structures/src/transitive_relation.rs
index 33ac279f3e0..31abea93819 100644
--- a/compiler/rustc_data_structures/src/transitive_relation.rs
+++ b/compiler/rustc_data_structures/src/transitive_relation.rs
@@ -354,6 +354,20 @@ impl<T: Eq + Hash + Copy> TransitiveRelation<T> {
             .collect()
     }
 
+    /// Given an element A, elements B with the lowest index such that `A R B`
+    /// and `B R A`, or `A` if no such element exists.
+    pub fn minimal_scc_representative(&self, a: T) -> T {
+        match self.index(a) {
+            Some(a_i) => self.with_closure(|closure| {
+                closure
+                    .iter(a_i.0)
+                    .find(|i| closure.contains(*i, a_i.0))
+                    .map_or(a, |i| self.elements[i])
+            }),
+            None => a,
+        }
+    }
+
     fn with_closure<OP, R>(&self, op: OP) -> R
     where
         OP: FnOnce(&BitMatrix<usize, usize>) -> R,
diff --git a/compiler/rustc_data_structures/src/transitive_relation/tests.rs b/compiler/rustc_data_structures/src/transitive_relation/tests.rs
index e756c546e41..cba14b5b64b 100644
--- a/compiler/rustc_data_structures/src/transitive_relation/tests.rs
+++ b/compiler/rustc_data_structures/src/transitive_relation/tests.rs
@@ -376,3 +376,44 @@ fn parent() {
     let p = relation.postdom_parent(3);
     assert_eq!(p, Some(0));
 }
+
+#[test]
+fn minimal_scc_representative_1() {
+    //      +---------+
+    //      v         |
+    // a -> c -> d -> e
+    //           ^    ^
+    //           |    |
+    //           b ---+
+
+    // "digraph { a -> c -> d -> e -> c; b -> d; b -> e; }",
+    let mut relation = TransitiveRelationBuilder::default();
+    relation.add("a", "c");
+    relation.add("c", "d");
+    relation.add("d", "e");
+    relation.add("e", "c");
+    relation.add("b", "d");
+    relation.add("b", "e");
+    let relation = relation.freeze();
+
+    assert_eq!(relation.minimal_scc_representative("a"), "a");
+    assert_eq!(relation.minimal_scc_representative("b"), "b");
+    assert_eq!(relation.minimal_scc_representative("c"), "c");
+    assert_eq!(relation.minimal_scc_representative("d"), "c");
+    assert_eq!(relation.minimal_scc_representative("e"), "c");
+}
+
+#[test]
+fn minimal_scc_representative_2() {
+    // "digraph { a -> b; a -> a; b -> a; c -> c}",
+    let mut relation = TransitiveRelationBuilder::default();
+    relation.add("a", "b");
+    relation.add("b", "a");
+    relation.add("a", "a");
+    relation.add("c", "c");
+    let relation = relation.freeze();
+
+    assert_eq!(relation.minimal_scc_representative("a"), "a");
+    assert_eq!(relation.minimal_scc_representative("b"), "a");
+    assert_eq!(relation.minimal_scc_representative("c"), "c");
+}
diff --git a/compiler/rustc_data_structures/src/vec_cache.rs b/compiler/rustc_data_structures/src/vec_cache.rs
index 2ff60ab7f36..3b448c056b7 100644
--- a/compiler/rustc_data_structures/src/vec_cache.rs
+++ b/compiler/rustc_data_structures/src/vec_cache.rs
@@ -68,22 +68,22 @@ impl SlotIndex {
     // slots (see `slot_index_exhaustive` in tests).
     #[inline]
     const fn from_index(idx: u32) -> Self {
-        let mut bucket = match idx.checked_ilog2() {
-            Some(x) => x as usize,
-            None => 0,
-        };
-        let entries;
-        let running_sum;
-        if bucket <= 11 {
-            entries = 1 << 12;
-            running_sum = 0;
-            bucket = 0;
-        } else {
-            entries = 1 << bucket;
-            running_sum = entries;
-            bucket = bucket - 11;
+        const FIRST_BUCKET_SHIFT: usize = 12;
+        if idx < (1 << FIRST_BUCKET_SHIFT) {
+            return SlotIndex {
+                bucket_idx: 0,
+                entries: 1 << FIRST_BUCKET_SHIFT,
+                index_in_bucket: idx as usize,
+            };
+        }
+        // SAFETY: We already ruled out idx 0, so `checked_ilog2` can't return `None`.
+        let bucket = unsafe { idx.checked_ilog2().unwrap_unchecked() as usize };
+        let entries = 1 << bucket;
+        SlotIndex {
+            bucket_idx: bucket - FIRST_BUCKET_SHIFT + 1,
+            entries,
+            index_in_bucket: idx as usize - entries,
         }
-        SlotIndex { bucket_idx: bucket, entries, index_in_bucket: idx as usize - running_sum }
     }
 
     // SAFETY: Buckets must be managed solely by functions here (i.e., get/put on SlotIndex) and
diff --git a/compiler/rustc_data_structures/src/vec_cache/tests.rs b/compiler/rustc_data_structures/src/vec_cache/tests.rs
index 3b65c14162e..9b60913ec92 100644
--- a/compiler/rustc_data_structures/src/vec_cache/tests.rs
+++ b/compiler/rustc_data_structures/src/vec_cache/tests.rs
@@ -75,24 +75,21 @@ fn slot_index_exhaustive() {
     for idx in 0..=u32::MAX {
         buckets[SlotIndex::from_index(idx).bucket_idx] += 1;
     }
-    let mut prev = None::<SlotIndex>;
-    for idx in 0..=u32::MAX {
+    let slot_idx = SlotIndex::from_index(0);
+    assert_eq!(slot_idx.index_in_bucket, 0);
+    assert_eq!(slot_idx.bucket_idx, 0);
+    let mut prev = slot_idx;
+    for idx in 1..=u32::MAX {
         let slot_idx = SlotIndex::from_index(idx);
-        if let Some(p) = prev {
-            if p.bucket_idx == slot_idx.bucket_idx {
-                assert_eq!(p.index_in_bucket + 1, slot_idx.index_in_bucket);
-            } else {
-                assert_eq!(slot_idx.index_in_bucket, 0);
-            }
+        if prev.bucket_idx == slot_idx.bucket_idx {
+            assert_eq!(prev.index_in_bucket + 1, slot_idx.index_in_bucket);
         } else {
-            assert_eq!(idx, 0);
             assert_eq!(slot_idx.index_in_bucket, 0);
-            assert_eq!(slot_idx.bucket_idx, 0);
         }
 
         assert_eq!(buckets[slot_idx.bucket_idx], slot_idx.entries as u32);
         assert_eq!(ENTRIES_BY_BUCKET[slot_idx.bucket_idx], slot_idx.entries, "{}", idx);
 
-        prev = Some(slot_idx);
+        prev = slot_idx;
     }
 }
diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs
index 056c476d5e1..54a331a4904 100644
--- a/compiler/rustc_driver_impl/src/lib.rs
+++ b/compiler/rustc_driver_impl/src/lib.rs
@@ -12,7 +12,6 @@
 #![feature(decl_macro)]
 #![feature(panic_backtrace_config)]
 #![feature(panic_update_hook)]
-#![feature(result_flattening)]
 #![feature(rustdoc_internals)]
 #![feature(try_blocks)]
 // tidy-alphabetical-end
diff --git a/compiler/rustc_error_codes/src/error_codes/E0092.md b/compiler/rustc_error_codes/src/error_codes/E0092.md
index be459d040c2..9c63798ded7 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0092.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0092.md
@@ -1,8 +1,10 @@
+#### Note: this error code is no longer emitted by the compiler.
+
 An undefined atomic operation function was declared.
 
 Erroneous code example:
 
-```compile_fail,E0092
+```ignore (no longer emitted)
 #![feature(intrinsics)]
 #![allow(internal_features)]
 
@@ -12,13 +14,4 @@ unsafe fn atomic_foo(); // error: unrecognized atomic operation
 ```
 
 Please check you didn't make a mistake in the function's name. All intrinsic
-functions are defined in `compiler/rustc_codegen_llvm/src/intrinsic.rs` and in
-`library/core/src/intrinsics.rs` in the Rust source code. Example:
-
-```
-#![feature(intrinsics)]
-#![allow(internal_features)]
-
-#[rustc_intrinsic]
-unsafe fn atomic_fence_seqcst(); // ok!
-```
+functions are defined in `library/core/src/intrinsics` in the Rust source code.
diff --git a/compiler/rustc_error_codes/src/error_codes/E0093.md b/compiler/rustc_error_codes/src/error_codes/E0093.md
index 9929a069927..3552c2db4cc 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0093.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0093.md
@@ -17,19 +17,4 @@ fn main() {
 ```
 
 Please check you didn't make a mistake in the function's name. All intrinsic
-functions are defined in `compiler/rustc_codegen_llvm/src/intrinsic.rs` and in
-`library/core/src/intrinsics.rs` in the Rust source code. Example:
-
-```
-#![feature(intrinsics)]
-#![allow(internal_features)]
-
-#[rustc_intrinsic]
-unsafe fn atomic_fence_seqcst(); // ok!
-
-fn main() {
-    unsafe {
-        atomic_fence_seqcst();
-    }
-}
-```
+functions are defined in `library/core/src/intrinsics` in the Rust source code.
diff --git a/compiler/rustc_error_codes/src/error_codes/E0622.md b/compiler/rustc_error_codes/src/error_codes/E0622.md
index 9b8131a061e..cc66e067990 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0622.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0622.md
@@ -4,7 +4,7 @@ An intrinsic was declared without being a function.
 
 Erroneous code example:
 
-```no_run
+```ignore (no longer emitted)
 #![feature(intrinsics)]
 #![allow(internal_features)]
 
@@ -21,7 +21,7 @@ An intrinsic is a function available for use in a given programming language
 whose implementation is handled specially by the compiler. In order to fix this
 error, just declare a function. Example:
 
-```no_run
+```ignore (no longer emitted)
 #![feature(intrinsics)]
 #![allow(internal_features)]
 
diff --git a/compiler/rustc_error_codes/src/error_codes/E0658.md b/compiler/rustc_error_codes/src/error_codes/E0658.md
index 24245a38ae0..65c82e4fb6e 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0658.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0658.md
@@ -3,10 +3,7 @@ An unstable feature was used.
 Erroneous code example:
 
 ```compile_fail,E0658
-#[repr(u128)] // error: use of unstable library feature 'repr128'
-enum Foo {
-    Bar(u64),
-}
+use std::intrinsics; // error: use of unstable library feature `core_intrinsics`
 ```
 
 If you're using a stable or a beta version of rustc, you won't be able to use
@@ -17,12 +14,9 @@ If you're using a nightly version of rustc, just add the corresponding feature
 to be able to use it:
 
 ```
-#![feature(repr128)]
+#![feature(core_intrinsics)]
 
-#[repr(u128)] // ok!
-enum Foo {
-    Bar(u64),
-}
+use std::intrinsics; // ok!
 ```
 
 [rustup]: https://rust-lang.github.io/rustup/concepts/channels.html
diff --git a/compiler/rustc_error_codes/src/error_codes/E0783.md b/compiler/rustc_error_codes/src/error_codes/E0783.md
index 73981e59e0d..ac8641cfc5a 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0783.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0783.md
@@ -9,7 +9,7 @@ match 2u8 {
 }
 ```
 
-Older Rust code using previous editions allowed `...` to stand for exclusive
+Older Rust code using previous editions allowed `...` to stand for inclusive
 ranges which are now signified using `..=`.
 
 To make this code compile replace the `...` with `..=`.
diff --git a/compiler/rustc_error_messages/Cargo.toml b/compiler/rustc_error_messages/Cargo.toml
index 578af7fc51d..5dc582b9c3a 100644
--- a/compiler/rustc_error_messages/Cargo.toml
+++ b/compiler/rustc_error_messages/Cargo.toml
@@ -5,8 +5,8 @@ edition = "2024"
 
 [dependencies]
 # tidy-alphabetical-start
-fluent-bundle = "0.15.2"
-fluent-syntax = "0.11"
+fluent-bundle = "0.16"
+fluent-syntax = "0.12"
 icu_list = "1.2"
 icu_locid = "1.2"
 icu_provider_adapters = "1.2"
@@ -16,6 +16,7 @@ rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_span = { path = "../rustc_span" }
+smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 tracing = "0.1"
 unic-langid = { version = "0.9.0", features = ["macros"] }
 # tidy-alphabetical-end
diff --git a/compiler/rustc_error_messages/src/lib.rs b/compiler/rustc_error_messages/src/lib.rs
index 3c6df147b1b..1d3b5b20751 100644
--- a/compiler/rustc_error_messages/src/lib.rs
+++ b/compiler/rustc_error_messages/src/lib.rs
@@ -21,6 +21,7 @@ use intl_memoizer::concurrent::IntlLangMemoizer;
 use rustc_data_structures::sync::IntoDynSyncSend;
 use rustc_macros::{Decodable, Encodable};
 use rustc_span::Span;
+use smallvec::SmallVec;
 use tracing::{instrument, trace};
 pub use unic_langid::{LanguageIdentifier, langid};
 
@@ -106,8 +107,7 @@ impl From<Vec<FluentError>> for TranslationBundleError {
 /// (overriding any conflicting messages).
 #[instrument(level = "trace")]
 pub fn fluent_bundle(
-    sysroot: PathBuf,
-    sysroot_candidates: Vec<PathBuf>,
+    sysroot_candidates: SmallVec<[PathBuf; 2]>,
     requested_locale: Option<LanguageIdentifier>,
     additional_ftl_path: Option<&Path>,
     with_directionality_markers: bool,
@@ -141,7 +141,7 @@ pub fn fluent_bundle(
     // If the user requests the default locale then don't try to load anything.
     if let Some(requested_locale) = requested_locale {
         let mut found_resources = false;
-        for mut sysroot in Some(sysroot).into_iter().chain(sysroot_candidates.into_iter()) {
+        for mut sysroot in sysroot_candidates {
             sysroot.push("share");
             sysroot.push("locale");
             sysroot.push(requested_locale.to_string());
diff --git a/compiler/rustc_errors/src/diagnostic.rs b/compiler/rustc_errors/src/diagnostic.rs
index 539ecfbb42e..a11f81b55bb 100644
--- a/compiler/rustc_errors/src/diagnostic.rs
+++ b/compiler/rustc_errors/src/diagnostic.rs
@@ -1325,7 +1325,7 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
             ));
             self.note("consider using `--verbose` to print the full type name to the console");
         }
-        Box::into_inner(self.diag.take().unwrap())
+        *self.diag.take().unwrap()
     }
 
     /// This method allows us to access the path of the file where "long types" are written to.
diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs
index f8e19e50778..133bd361ee7 100644
--- a/compiler/rustc_errors/src/lib.rs
+++ b/compiler/rustc_errors/src/lib.rs
@@ -12,16 +12,13 @@
 #![feature(array_windows)]
 #![feature(assert_matches)]
 #![feature(associated_type_defaults)]
-#![feature(box_into_inner)]
 #![feature(box_patterns)]
 #![feature(default_field_values)]
 #![feature(error_reporter)]
-#![feature(if_let_guard)]
 #![feature(negative_impls)]
 #![feature(never_type)]
 #![feature(rustc_attrs)]
 #![feature(rustdoc_internals)]
-#![feature(trait_alias)]
 #![feature(try_blocks)]
 #![feature(yeet_expr)]
 // tidy-alphabetical-end
@@ -1530,7 +1527,7 @@ impl DiagCtxtInner {
             // Future breakages aren't emitted if they're `Level::Allow` or
             // `Level::Expect`, but they still need to be constructed and
             // stashed below, so they'll trigger the must_produce_diag check.
-            assert_matches!(diagnostic.level, Error | Warning | Allow | Expect);
+            assert_matches!(diagnostic.level, Error | ForceWarning | Warning | Allow | Expect);
             self.future_breakage_diagnostics.push(diagnostic.clone());
         }
 
diff --git a/compiler/rustc_expand/Cargo.toml b/compiler/rustc_expand/Cargo.toml
index e8fd2f54d76..57dd3a3128d 100644
--- a/compiler/rustc_expand/Cargo.toml
+++ b/compiler/rustc_expand/Cargo.toml
@@ -23,6 +23,9 @@ rustc_lexer = { path = "../rustc_lexer" }
 rustc_lint_defs = { path = "../rustc_lint_defs" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_parse = { path = "../rustc_parse" }
+# We must use the proc_macro version that we will compile proc-macros against,
+# not the one from our own sysroot.
+rustc_proc_macro = { path = "../rustc_proc_macro" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs
index 55751aa4908..c7b975d8f3e 100644
--- a/compiler/rustc_expand/src/base.rs
+++ b/compiler/rustc_expand/src/base.rs
@@ -167,7 +167,7 @@ impl Annotatable {
 
     pub fn expect_stmt(self) -> ast::Stmt {
         match self {
-            Annotatable::Stmt(stmt) => stmt.into_inner(),
+            Annotatable::Stmt(stmt) => *stmt,
             _ => panic!("expected statement"),
         }
     }
@@ -1424,7 +1424,7 @@ pub fn parse_macro_name_and_helper_attrs(
 /// See #73345 and #83125 for more details.
 /// FIXME(#73933): Remove this eventually.
 fn pretty_printing_compatibility_hack(item: &Item, psess: &ParseSess) {
-    if let ast::ItemKind::Enum(ident, enum_def, _) = &item.kind
+    if let ast::ItemKind::Enum(ident, _, enum_def) = &item.kind
         && ident.name == sym::ProceduralMasqueradeDummyType
         && let [variant] = &*enum_def.variants
         && variant.ident.name == sym::Input
diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs
index 0994813ecb9..c50ab5959e2 100644
--- a/compiler/rustc_expand/src/config.rs
+++ b/compiler/rustc_expand/src/config.rs
@@ -2,7 +2,6 @@
 
 use std::iter;
 
-use rustc_ast::ptr::P;
 use rustc_ast::token::{Delimiter, Token, TokenKind};
 use rustc_ast::tokenstream::{
     AttrTokenStream, AttrTokenTree, LazyAttrTokenStream, Spacing, TokenTree,
@@ -433,7 +432,7 @@ impl<'a> StripUnconfigured<'a> {
     }
 
     #[instrument(level = "trace", skip(self))]
-    pub fn configure_expr(&self, expr: &mut P<ast::Expr>, method_receiver: bool) {
+    pub fn configure_expr(&self, expr: &mut ast::Expr, method_receiver: bool) {
         if !method_receiver {
             for attr in expr.attrs.iter() {
                 self.maybe_emit_expr_attr_err(attr);
diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs
index 81d4d59ee04..569165a64e5 100644
--- a/compiler/rustc_expand/src/expand.rs
+++ b/compiler/rustc_expand/src/expand.rs
@@ -3,15 +3,14 @@ use std::rc::Rc;
 use std::sync::Arc;
 use std::{iter, mem};
 
-use rustc_ast as ast;
 use rustc_ast::mut_visit::*;
 use rustc_ast::ptr::P;
 use rustc_ast::tokenstream::TokenStream;
 use rustc_ast::visit::{self, AssocCtxt, Visitor, VisitorResult, try_visit, walk_list};
 use rustc_ast::{
-    AssocItemKind, AstNodeWrapper, AttrArgs, AttrStyle, AttrVec, ExprKind, ForeignItemKind,
-    HasAttrs, HasNodeId, Inline, ItemKind, MacStmtStyle, MetaItemInner, MetaItemKind, ModKind,
-    NodeId, PatKind, StmtKind, TyKind, token,
+    self as ast, AssocItemKind, AstNodeWrapper, AttrArgs, AttrStyle, AttrVec, DUMMY_NODE_ID,
+    ExprKind, ForeignItemKind, HasAttrs, HasNodeId, Inline, ItemKind, MacStmtStyle, MetaItemInner,
+    MetaItemKind, ModKind, NodeId, PatKind, StmtKind, TyKind, token,
 };
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::flat_map_in_place::FlatMapInPlace;
@@ -131,13 +130,9 @@ macro_rules! ast_fragments {
             pub(crate) fn mut_visit_with<F: MutVisitor>(&mut self, vis: &mut F) {
                 match self {
                     AstFragment::OptExpr(opt_expr) => {
-                        visit_clobber(opt_expr, |opt_expr| {
-                            if let Some(expr) = opt_expr {
-                                vis.filter_map_expr(expr)
-                            } else {
-                                None
-                            }
-                        });
+                        if let Some(expr) = opt_expr.take() {
+                            *opt_expr = vis.filter_map_expr(expr)
+                        }
                     }
                     AstFragment::MethodReceiverExpr(expr) => vis.visit_method_receiver_expr(expr),
                     $($(AstFragment::$Kind(ast) => vis.$mut_visit_ast(ast),)?)*
@@ -1188,9 +1183,8 @@ impl InvocationCollectorNode for P<ast::Item> {
         matches!(self.kind, ItemKind::MacCall(..))
     }
     fn take_mac_call(self) -> (P<ast::MacCall>, ast::AttrVec, AddSemicolon) {
-        let node = self.into_inner();
-        match node.kind {
-            ItemKind::MacCall(mac) => (mac, node.attrs, AddSemicolon::No),
+        match self.kind {
+            ItemKind::MacCall(mac) => (mac, self.attrs, AddSemicolon::No),
             _ => unreachable!(),
         }
     }
@@ -1319,10 +1313,10 @@ impl InvocationCollectorNode for P<ast::Item> {
 
             let mut idents = Vec::new();
             collect_use_tree_leaves(ut, &mut idents);
-            return idents;
+            idents
+        } else {
+            self.kind.ident().into_iter().collect()
         }
-
-        if let Some(ident) = self.kind.ident() { vec![ident] } else { vec![] }
     }
 }
 
@@ -1344,7 +1338,7 @@ impl InvocationCollectorNode for AstNodeWrapper<P<ast::AssocItem>, TraitItemTag>
         matches!(self.wrapped.kind, AssocItemKind::MacCall(..))
     }
     fn take_mac_call(self) -> (P<ast::MacCall>, ast::AttrVec, AddSemicolon) {
-        let item = self.wrapped.into_inner();
+        let item = self.wrapped;
         match item.kind {
             AssocItemKind::MacCall(mac) => (mac, item.attrs, AddSemicolon::No),
             _ => unreachable!(),
@@ -1385,7 +1379,7 @@ impl InvocationCollectorNode for AstNodeWrapper<P<ast::AssocItem>, ImplItemTag>
         matches!(self.wrapped.kind, AssocItemKind::MacCall(..))
     }
     fn take_mac_call(self) -> (P<ast::MacCall>, ast::AttrVec, AddSemicolon) {
-        let item = self.wrapped.into_inner();
+        let item = self.wrapped;
         match item.kind {
             AssocItemKind::MacCall(mac) => (mac, item.attrs, AddSemicolon::No),
             _ => unreachable!(),
@@ -1426,7 +1420,7 @@ impl InvocationCollectorNode for AstNodeWrapper<P<ast::AssocItem>, TraitImplItem
         matches!(self.wrapped.kind, AssocItemKind::MacCall(..))
     }
     fn take_mac_call(self) -> (P<ast::MacCall>, ast::AttrVec, AddSemicolon) {
-        let item = self.wrapped.into_inner();
+        let item = self.wrapped;
         match item.kind {
             AssocItemKind::MacCall(mac) => (mac, item.attrs, AddSemicolon::No),
             _ => unreachable!(),
@@ -1464,9 +1458,8 @@ impl InvocationCollectorNode for P<ast::ForeignItem> {
         matches!(self.kind, ForeignItemKind::MacCall(..))
     }
     fn take_mac_call(self) -> (P<ast::MacCall>, ast::AttrVec, AddSemicolon) {
-        let node = self.into_inner();
-        match node.kind {
-            ForeignItemKind::MacCall(mac) => (mac, node.attrs, AddSemicolon::No),
+        match self.kind {
+            ForeignItemKind::MacCall(mac) => (mac, self.attrs, AddSemicolon::No),
             _ => unreachable!(),
         }
     }
@@ -1601,16 +1594,16 @@ impl InvocationCollectorNode for ast::Stmt {
         // `StmtKind`s and treat them as statement macro invocations, not as items or expressions.
         let (add_semicolon, mac, attrs) = match self.kind {
             StmtKind::MacCall(mac) => {
-                let ast::MacCallStmt { mac, style, attrs, .. } = mac.into_inner();
+                let ast::MacCallStmt { mac, style, attrs, .. } = *mac;
                 (style == MacStmtStyle::Semicolon, mac, attrs)
             }
-            StmtKind::Item(item) => match item.into_inner() {
+            StmtKind::Item(item) => match *item {
                 ast::Item { kind: ItemKind::MacCall(mac), attrs, .. } => {
                     (mac.args.need_semicolon(), mac, attrs)
                 }
                 _ => unreachable!(),
             },
-            StmtKind::Semi(expr) => match expr.into_inner() {
+            StmtKind::Semi(expr) => match *expr {
                 ast::Expr { kind: ExprKind::MacCall(mac), attrs, .. } => {
                     (mac.args.need_semicolon(), mac, attrs)
                 }
@@ -1691,8 +1684,7 @@ impl InvocationCollectorNode for P<ast::Ty> {
         matches!(self.kind, ast::TyKind::MacCall(..))
     }
     fn take_mac_call(self) -> (P<ast::MacCall>, ast::AttrVec, AddSemicolon) {
-        let node = self.into_inner();
-        match node.kind {
+        match self.kind {
             TyKind::MacCall(mac) => (mac, AttrVec::new(), AddSemicolon::No),
             _ => unreachable!(),
         }
@@ -1715,8 +1707,7 @@ impl InvocationCollectorNode for P<ast::Pat> {
         matches!(self.kind, PatKind::MacCall(..))
     }
     fn take_mac_call(self) -> (P<ast::MacCall>, ast::AttrVec, AddSemicolon) {
-        let node = self.into_inner();
-        match node.kind {
+        match self.kind {
             PatKind::MacCall(mac) => (mac, AttrVec::new(), AddSemicolon::No),
             _ => unreachable!(),
         }
@@ -1742,9 +1733,8 @@ impl InvocationCollectorNode for P<ast::Expr> {
         matches!(self.kind, ExprKind::MacCall(..))
     }
     fn take_mac_call(self) -> (P<ast::MacCall>, ast::AttrVec, AddSemicolon) {
-        let node = self.into_inner();
-        match node.kind {
-            ExprKind::MacCall(mac) => (mac, node.attrs, AddSemicolon::No),
+        match self.kind {
+            ExprKind::MacCall(mac) => (mac, self.attrs, AddSemicolon::No),
             _ => unreachable!(),
         }
     }
@@ -1768,7 +1758,7 @@ impl InvocationCollectorNode for AstNodeWrapper<P<ast::Expr>, OptExprTag> {
         matches!(self.wrapped.kind, ast::ExprKind::MacCall(..))
     }
     fn take_mac_call(self) -> (P<ast::MacCall>, ast::AttrVec, AddSemicolon) {
-        let node = self.wrapped.into_inner();
+        let node = self.wrapped;
         match node.kind {
             ExprKind::MacCall(mac) => (mac, node.attrs, AddSemicolon::No),
             _ => unreachable!(),
@@ -1782,11 +1772,7 @@ impl InvocationCollectorNode for AstNodeWrapper<P<ast::Expr>, OptExprTag> {
 /// This struct is a hack to workaround unstable of `stmt_expr_attributes`.
 /// It can be removed once that feature is stabilized.
 struct MethodReceiverTag;
-impl DummyAstNode for MethodReceiverTag {
-    fn dummy() -> MethodReceiverTag {
-        MethodReceiverTag
-    }
-}
+
 impl InvocationCollectorNode for AstNodeWrapper<P<ast::Expr>, MethodReceiverTag> {
     type OutputTy = Self;
     const KIND: AstFragmentKind = AstFragmentKind::MethodReceiverExpr;
@@ -1806,7 +1792,7 @@ impl InvocationCollectorNode for AstNodeWrapper<P<ast::Expr>, MethodReceiverTag>
         matches!(self.wrapped.kind, ast::ExprKind::MacCall(..))
     }
     fn take_mac_call(self) -> (P<ast::MacCall>, ast::AttrVec, AddSemicolon) {
-        let node = self.wrapped.into_inner();
+        let node = self.wrapped;
         match node.kind {
             ExprKind::MacCall(mac) => (mac, node.attrs, AddSemicolon::No),
             _ => unreachable!(),
@@ -1852,6 +1838,57 @@ fn build_single_delegations<'a, Node: InvocationCollectorNode>(
     })
 }
 
+/// Required for `visit_node` obtained an owned `Node` from `&mut Node`.
+trait DummyAstNode {
+    fn dummy() -> Self;
+}
+
+impl DummyAstNode for ast::Crate {
+    fn dummy() -> Self {
+        ast::Crate {
+            attrs: Default::default(),
+            items: Default::default(),
+            spans: Default::default(),
+            id: DUMMY_NODE_ID,
+            is_placeholder: Default::default(),
+        }
+    }
+}
+
+impl DummyAstNode for P<ast::Ty> {
+    fn dummy() -> Self {
+        P(ast::Ty {
+            id: DUMMY_NODE_ID,
+            kind: TyKind::Dummy,
+            span: Default::default(),
+            tokens: Default::default(),
+        })
+    }
+}
+
+impl DummyAstNode for P<ast::Pat> {
+    fn dummy() -> Self {
+        P(ast::Pat {
+            id: DUMMY_NODE_ID,
+            kind: PatKind::Wild,
+            span: Default::default(),
+            tokens: Default::default(),
+        })
+    }
+}
+
+impl DummyAstNode for P<ast::Expr> {
+    fn dummy() -> Self {
+        ast::Expr::dummy()
+    }
+}
+
+impl DummyAstNode for AstNodeWrapper<P<ast::Expr>, MethodReceiverTag> {
+    fn dummy() -> Self {
+        AstNodeWrapper::new(ast::Expr::dummy(), MethodReceiverTag)
+    }
+}
+
 struct InvocationCollector<'a, 'b> {
     cx: &'a mut ExtCtxt<'b>,
     invocations: Vec<(Invocation, Option<Arc<SyntaxExtension>>)>,
@@ -2155,18 +2192,19 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
                         self.expand_cfg_attr(node, &attr, pos);
                         continue;
                     }
-                    _ => visit_clobber(node, |node| {
-                        self.collect_attr((attr, pos, derives), node.to_annotatable(), Node::KIND)
+                    _ => {
+                        let n = mem::replace(node, Node::dummy());
+                        *node = self
+                            .collect_attr((attr, pos, derives), n.to_annotatable(), Node::KIND)
                             .make_ast::<Node>()
-                    }),
+                    }
                 },
                 None if node.is_mac_call() => {
-                    visit_clobber(node, |node| {
-                        // Do not clobber unless it's actually a macro (uncommon case).
-                        let (mac, attrs, _) = node.take_mac_call();
-                        self.check_attributes(&attrs, &mac);
-                        self.collect_bang(mac, Node::KIND).make_ast::<Node>()
-                    })
+                    let n = mem::replace(node, Node::dummy());
+                    let (mac, attrs, _) = n.take_mac_call();
+                    self.check_attributes(&attrs, &mac);
+
+                    *node = self.collect_bang(mac, Node::KIND).make_ast::<Node>()
                 }
                 None if node.delegation().is_some() => unreachable!(),
                 None => {
@@ -2293,18 +2331,14 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
     }
 
     fn visit_method_receiver_expr(&mut self, node: &mut P<ast::Expr>) {
-        visit_clobber(node, |node| {
-            let mut wrapper = AstNodeWrapper::new(node, MethodReceiverTag);
-            self.visit_node(&mut wrapper);
-            wrapper.wrapped
-        })
+        self.visit_node(AstNodeWrapper::from_mut(node, MethodReceiverTag))
     }
 
     fn filter_map_expr(&mut self, node: P<ast::Expr>) -> Option<P<ast::Expr>> {
         self.flat_map_node(AstNodeWrapper::new(node, OptExprTag))
     }
 
-    fn visit_block(&mut self, node: &mut P<ast::Block>) {
+    fn visit_block(&mut self, node: &mut ast::Block) {
         let orig_dir_ownership = mem::replace(
             &mut self.cx.current_expansion.dir_ownership,
             DirOwnership::UnownedViaBlock,
diff --git a/compiler/rustc_expand/src/lib.rs b/compiler/rustc_expand/src/lib.rs
index cd744977bb3..515d82296ca 100644
--- a/compiler/rustc_expand/src/lib.rs
+++ b/compiler/rustc_expand/src/lib.rs
@@ -6,7 +6,6 @@
 #![feature(associated_type_defaults)]
 #![feature(if_let_guard)]
 #![feature(macro_metavar_expr)]
-#![feature(map_try_insert)]
 #![feature(proc_macro_diagnostic)]
 #![feature(proc_macro_internals)]
 #![feature(rustdoc_internals)]
@@ -14,8 +13,6 @@
 #![feature(yeet_expr)]
 // tidy-alphabetical-end
 
-extern crate proc_macro as pm;
-
 mod build;
 mod errors;
 // FIXME(Nilstrieb) Translate macro_rules diagnostics
diff --git a/compiler/rustc_expand/src/placeholders.rs b/compiler/rustc_expand/src/placeholders.rs
index 0136292decb..3dcb20c8c76 100644
--- a/compiler/rustc_expand/src/placeholders.rs
+++ b/compiler/rustc_expand/src/placeholders.rs
@@ -349,7 +349,7 @@ impl MutVisitor for PlaceholderExpander {
     fn filter_map_expr(&mut self, expr: P<ast::Expr>) -> Option<P<ast::Expr>> {
         match expr.kind {
             ast::ExprKind::MacCall(_) => self.remove(expr.id).make_opt_expr(),
-            _ => noop_filter_map_expr(self, expr),
+            _ => walk_filter_map_expr(self, expr),
         }
     }
 
diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs
index d5af9849e75..84fbbbef061 100644
--- a/compiler/rustc_expand/src/proc_macro.rs
+++ b/compiler/rustc_expand/src/proc_macro.rs
@@ -1,4 +1,3 @@
-use rustc_ast as ast;
 use rustc_ast::ptr::P;
 use rustc_ast::tokenstream::TokenStream;
 use rustc_errors::ErrorGuaranteed;
@@ -6,6 +5,7 @@ use rustc_parse::parser::{ForceCollect, Parser};
 use rustc_session::config::ProcMacroExecutionStrategy;
 use rustc_span::Span;
 use rustc_span::profiling::SpannedEventArgRecorder;
+use {rustc_ast as ast, rustc_proc_macro as pm};
 
 use crate::base::{self, *};
 use crate::{errors, proc_macro_server};
diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs
index f00201ad202..fb5abaefb57 100644
--- a/compiler/rustc_expand/src/proc_macro_server.rs
+++ b/compiler/rustc_expand/src/proc_macro_server.rs
@@ -1,10 +1,6 @@
 use std::ops::{Bound, Range};
 
 use ast::token::IdentIsRaw;
-use pm::bridge::{
-    DelimSpan, Diagnostic, ExpnGlobals, Group, Ident, LitKind, Literal, Punct, TokenTree, server,
-};
-use pm::{Delimiter, Level};
 use rustc_ast as ast;
 use rustc_ast::token;
 use rustc_ast::tokenstream::{self, DelimSpacing, Spacing, TokenStream};
@@ -15,6 +11,10 @@ use rustc_errors::{Diag, ErrorGuaranteed, MultiSpan, PResult};
 use rustc_parse::lexer::nfc_normalize;
 use rustc_parse::parser::Parser;
 use rustc_parse::{exp, new_parser_from_source_str, source_str_to_stream, unwrap_or_emit_fatal};
+use rustc_proc_macro::bridge::{
+    DelimSpan, Diagnostic, ExpnGlobals, Group, Ident, LitKind, Literal, Punct, TokenTree, server,
+};
+use rustc_proc_macro::{Delimiter, Level};
 use rustc_session::parse::ParseSess;
 use rustc_span::def_id::CrateNum;
 use rustc_span::{BytePos, FileName, Pos, Span, Symbol, sym};
@@ -66,7 +66,7 @@ impl FromInternal<token::LitKind> for LitKind {
             token::CStr => LitKind::CStr,
             token::CStrRaw(n) => LitKind::CStrRaw(n),
             token::Err(_guar) => {
-                // This is the only place a `pm::bridge::LitKind::ErrWithGuar`
+                // This is the only place a `rustc_proc_macro::bridge::LitKind::ErrWithGuar`
                 // is constructed. Note that an `ErrorGuaranteed` is available,
                 // as required. See the comment in `to_internal`.
                 LitKind::ErrWithGuar
@@ -149,7 +149,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
                     }
 
                     trees.push(TokenTree::Group(Group {
-                        delimiter: pm::Delimiter::from_internal(delim),
+                        delimiter: rustc_proc_macro::Delimiter::from_internal(delim),
                         stream: Some(stream),
                         span: DelimSpan {
                             open: span.open,
@@ -270,7 +270,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
                     let stream =
                         TokenStream::token_alone(token::Lifetime(ident.name, is_raw), ident.span);
                     trees.push(TokenTree::Group(Group {
-                        delimiter: pm::Delimiter::None,
+                        delimiter: rustc_proc_macro::Delimiter::None,
                         stream: Some(stream),
                         span: DelimSpan::from_single(span),
                     }))
@@ -302,7 +302,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
                         trees.push(TokenTree::Punct(Punct { ch: b'!', joint: false, span }));
                     }
                     trees.push(TokenTree::Group(Group {
-                        delimiter: pm::Delimiter::Bracket,
+                        delimiter: rustc_proc_macro::Delimiter::Bracket,
                         stream: Some(stream),
                         span: DelimSpan::from_single(span),
                     }));
diff --git a/compiler/rustc_feature/src/accepted.rs b/compiler/rustc_feature/src/accepted.rs
index 820af9ac84b..ffa6ffb40b6 100644
--- a/compiler/rustc_feature/src/accepted.rs
+++ b/compiler/rustc_feature/src/accepted.rs
@@ -360,6 +360,8 @@ declare_features! (
     (accepted, relaxed_adts, "1.19.0", Some(35626)),
     /// Lessens the requirements for structs to implement `Unsize`.
     (accepted, relaxed_struct_unsize, "1.58.0", Some(81793)),
+    /// Allows the `#[repr(i128)]` attribute for enums.
+    (accepted, repr128, "CURRENT_RUSTC_VERSION", Some(56071)),
     /// Allows `repr(align(16))` struct attribute (RFC 1358).
     (accepted, repr_align, "1.25.0", Some(33626)),
     /// Allows using `#[repr(align(X))]` on enums with equivalent semantics
diff --git a/compiler/rustc_feature/src/removed.rs b/compiler/rustc_feature/src/removed.rs
index 687d859df53..013e1d5d0fa 100644
--- a/compiler/rustc_feature/src/removed.rs
+++ b/compiler/rustc_feature/src/removed.rs
@@ -207,6 +207,8 @@ declare_features! (
     /// Allows exhaustive integer pattern matching with `usize::MAX`/`isize::MIN`/`isize::MAX`.
     (removed, precise_pointer_size_matching, "1.32.0", Some(56354),
      Some("removed in favor of half-open ranges")),
+    (removed, pref_align_of, "CURRENT_RUSTC_VERSION", Some(91971),
+     Some("removed due to marginal use and inducing compiler complications")),
     (removed, proc_macro_expr, "1.27.0", Some(54727),
      Some("subsumed by `#![feature(proc_macro_hygiene)]`")),
     (removed, proc_macro_gen, "1.27.0", Some(54727),
diff --git a/compiler/rustc_feature/src/unstable.rs b/compiler/rustc_feature/src/unstable.rs
index 6cdcf451f37..b46eac6d8a6 100644
--- a/compiler/rustc_feature/src/unstable.rs
+++ b/compiler/rustc_feature/src/unstable.rs
@@ -371,6 +371,8 @@ declare_features! (
     (unstable, arbitrary_self_types, "1.23.0", Some(44874)),
     /// Allows inherent and trait methods with arbitrary self types that are raw pointers.
     (unstable, arbitrary_self_types_pointers, "1.83.0", Some(44874)),
+    /// Allows #[cfg(...)] on inline assembly templates and operands.
+    (unstable, asm_cfg, "CURRENT_RUSTC_VERSION", Some(140364)),
     /// Enables experimental inline assembly support for additional architectures.
     (unstable, asm_experimental_arch, "1.58.0", Some(93335)),
     /// Enables experimental register support in inline assembly.
@@ -619,8 +621,6 @@ declare_features! (
     (incomplete, ref_pat_eat_one_layer_2024_structural, "1.81.0", Some(123076)),
     /// Allows using the `#[register_tool]` attribute.
     (unstable, register_tool, "1.41.0", Some(66079)),
-    /// Allows the `#[repr(i128)]` attribute for enums.
-    (incomplete, repr128, "1.16.0", Some(56071)),
     /// Allows `repr(simd)` and importing the various simd intrinsics.
     (unstable, repr_simd, "1.4.0", Some(27731)),
     /// Allows bounding the return type of AFIT/RPITIT.
diff --git a/compiler/rustc_fluent_macro/Cargo.toml b/compiler/rustc_fluent_macro/Cargo.toml
index ce76b2745ea..d7ef4280aef 100644
--- a/compiler/rustc_fluent_macro/Cargo.toml
+++ b/compiler/rustc_fluent_macro/Cargo.toml
@@ -9,8 +9,8 @@ proc-macro = true
 [dependencies]
 # tidy-alphabetical-start
 annotate-snippets = "0.11"
-fluent-bundle = "0.15.2"
-fluent-syntax = "0.11"
+fluent-bundle = "0.16"
+fluent-syntax = "0.12"
 proc-macro2 = "1"
 quote = "1"
 syn = { version = "2", features = ["full"] }
diff --git a/compiler/rustc_fluent_macro/src/lib.rs b/compiler/rustc_fluent_macro/src/lib.rs
index c6e0484b921..6f85e05f29a 100644
--- a/compiler/rustc_fluent_macro/src/lib.rs
+++ b/compiler/rustc_fluent_macro/src/lib.rs
@@ -4,7 +4,6 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(proc_macro_diagnostic)]
-#![feature(proc_macro_span)]
 #![feature(rustdoc_internals)]
 #![feature(track_path)]
 // tidy-alphabetical-end
diff --git a/compiler/rustc_hir/src/arena.rs b/compiler/rustc_hir/src/arena.rs
index 88c0d223fd3..b0dff635a9b 100644
--- a/compiler/rustc_hir/src/arena.rs
+++ b/compiler/rustc_hir/src/arena.rs
@@ -8,7 +8,6 @@ macro_rules! arena_types {
             [] asm_template: rustc_ast::InlineAsmTemplatePiece,
             [] attribute: rustc_hir::Attribute,
             [] owner_info: rustc_hir::OwnerInfo<'tcx>,
-            [] use_path: rustc_hir::UsePath<'tcx>,
             [] lit: rustc_hir::Lit,
             [] macro_def: rustc_ast::MacroDef,
         ]);
diff --git a/compiler/rustc_hir/src/def.rs b/compiler/rustc_hir/src/def.rs
index 507c94aca8b..459fe5935e0 100644
--- a/compiler/rustc_hir/src/def.rs
+++ b/compiler/rustc_hir/src/def.rs
@@ -584,7 +584,7 @@ impl<CTX: crate::HashStableContext> ToStableHashKey<CTX> for Namespace {
 }
 
 /// Just a helper ‒ separate structure for each namespace.
-#[derive(Copy, Clone, Default, Debug)]
+#[derive(Copy, Clone, Default, Debug, HashStable_Generic)]
 pub struct PerNS<T> {
     pub value_ns: T,
     pub type_ns: T,
@@ -596,10 +596,16 @@ impl<T> PerNS<T> {
         PerNS { value_ns: f(self.value_ns), type_ns: f(self.type_ns), macro_ns: f(self.macro_ns) }
     }
 
+    /// Note: Do you really want to use this? Often you know which namespace a
+    /// name will belong in, and you can consider just that namespace directly,
+    /// rather than iterating through all of them.
     pub fn into_iter(self) -> IntoIter<T, 3> {
         [self.value_ns, self.type_ns, self.macro_ns].into_iter()
     }
 
+    /// Note: Do you really want to use this? Often you know which namespace a
+    /// name will belong in, and you can consider just that namespace directly,
+    /// rather than iterating through all of them.
     pub fn iter(&self) -> IntoIter<&T, 3> {
         [&self.value_ns, &self.type_ns, &self.macro_ns].into_iter()
     }
@@ -634,6 +640,10 @@ impl<T> PerNS<Option<T>> {
     }
 
     /// Returns an iterator over the items which are `Some`.
+    ///
+    /// Note: Do you really want to use this? Often you know which namespace a
+    /// name will belong in, and you can consider just that namespace directly,
+    /// rather than iterating through all of them.
     pub fn present_items(self) -> impl Iterator<Item = T> {
         [self.type_ns, self.value_ns, self.macro_ns].into_iter().flatten()
     }
@@ -842,12 +852,7 @@ pub enum LifetimeRes {
     /// late resolution. Those lifetimes will be inferred by typechecking.
     Infer,
     /// `'static` lifetime.
-    Static {
-        /// We do not want to emit `elided_named_lifetimes`
-        /// when we are inside of a const item or a static,
-        /// because it would get too annoying.
-        suppress_elision_warning: bool,
-    },
+    Static,
     /// Resolution failure.
     Error,
     /// HACK: This is used to recover the NodeId of an elided lifetime.
diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs
index fa1d1ec0a86..6f288bb39b9 100644
--- a/compiler/rustc_hir/src/hir.rs
+++ b/compiler/rustc_hir/src/hir.rs
@@ -30,7 +30,7 @@ use thin_vec::ThinVec;
 use tracing::debug;
 
 use crate::LangItem;
-use crate::def::{CtorKind, DefKind, Res};
+use crate::def::{CtorKind, DefKind, PerNS, Res};
 use crate::def_id::{DefId, LocalDefIdMap};
 pub(crate) use crate::hir_id::{HirId, ItemLocalId, ItemLocalMap, OwnerId};
 use crate::intravisit::{FnKind, VisitorExt};
@@ -72,26 +72,26 @@ pub enum LifetimeSource {
 #[derive(Debug, Copy, Clone, PartialEq, Eq, HashStable_Generic)]
 pub enum LifetimeSyntax {
     /// E.g. `&Type`, `ContainsLifetime`
-    Hidden,
+    Implicit,
 
     /// E.g. `&'_ Type`, `ContainsLifetime<'_>`, `impl Trait + '_`, `impl Trait + use<'_>`
-    Anonymous,
+    ExplicitAnonymous,
 
     /// E.g. `&'a Type`, `ContainsLifetime<'a>`, `impl Trait + 'a`, `impl Trait + use<'a>`
-    Named,
+    ExplicitBound,
 }
 
 impl From<Ident> for LifetimeSyntax {
     fn from(ident: Ident) -> Self {
         let name = ident.name;
 
-        if name == kw::Empty {
+        if name == sym::empty {
             unreachable!("A lifetime name should never be empty");
         } else if name == kw::UnderscoreLifetime {
-            LifetimeSyntax::Anonymous
+            LifetimeSyntax::ExplicitAnonymous
         } else {
             debug_assert!(name.as_str().starts_with('\''));
-            LifetimeSyntax::Named
+            LifetimeSyntax::ExplicitBound
         }
     }
 }
@@ -102,48 +102,48 @@ impl From<Ident> for LifetimeSyntax {
 ///
 /// ```
 /// #[repr(C)]
-/// struct S<'a>(&'a u32);       // res=Param, name='a, source=Reference, syntax=Named
+/// struct S<'a>(&'a u32);       // res=Param, name='a, source=Reference, syntax=ExplicitBound
 /// unsafe extern "C" {
-///     fn f1(s: S);             // res=Param, name='_, source=Path, syntax=Hidden
-///     fn f2(s: S<'_>);         // res=Param, name='_, source=Path, syntax=Anonymous
-///     fn f3<'a>(s: S<'a>);     // res=Param, name='a, source=Path, syntax=Named
+///     fn f1(s: S);             // res=Param, name='_, source=Path, syntax=Implicit
+///     fn f2(s: S<'_>);         // res=Param, name='_, source=Path, syntax=ExplicitAnonymous
+///     fn f3<'a>(s: S<'a>);     // res=Param, name='a, source=Path, syntax=ExplicitBound
 /// }
 ///
-/// struct St<'a> { x: &'a u32 } // res=Param, name='a, source=Reference, syntax=Named
+/// struct St<'a> { x: &'a u32 } // res=Param, name='a, source=Reference, syntax=ExplicitBound
 /// fn f() {
-///     _ = St { x: &0 };        // res=Infer, name='_, source=Path, syntax=Hidden
-///     _ = St::<'_> { x: &0 };  // res=Infer, name='_, source=Path, syntax=Anonymous
+///     _ = St { x: &0 };        // res=Infer, name='_, source=Path, syntax=Implicit
+///     _ = St::<'_> { x: &0 };  // res=Infer, name='_, source=Path, syntax=ExplicitAnonymous
 /// }
 ///
-/// struct Name<'a>(&'a str);    // res=Param,  name='a, source=Reference, syntax=Named
-/// const A: Name = Name("a");   // res=Static, name='_, source=Path, syntax=Hidden
-/// const B: &str = "";          // res=Static, name='_, source=Reference, syntax=Hidden
-/// static C: &'_ str = "";      // res=Static, name='_, source=Reference, syntax=Anonymous
-/// static D: &'static str = ""; // res=Static, name='static, source=Reference, syntax=Named
+/// struct Name<'a>(&'a str);    // res=Param,  name='a, source=Reference, syntax=ExplicitBound
+/// const A: Name = Name("a");   // res=Static, name='_, source=Path, syntax=Implicit
+/// const B: &str = "";          // res=Static, name='_, source=Reference, syntax=Implicit
+/// static C: &'_ str = "";      // res=Static, name='_, source=Reference, syntax=ExplicitAnonymous
+/// static D: &'static str = ""; // res=Static, name='static, source=Reference, syntax=ExplicitBound
 ///
 /// trait Tr {}
-/// fn tr(_: Box<dyn Tr>) {}     // res=ImplicitObjectLifetimeDefault, name='_, source=Other, syntax=Hidden
+/// fn tr(_: Box<dyn Tr>) {}     // res=ImplicitObjectLifetimeDefault, name='_, source=Other, syntax=Implicit
 ///
 /// fn capture_outlives<'a>() ->
-///     impl FnOnce() + 'a       // res=Param, ident='a, source=OutlivesBound, syntax=Named
+///     impl FnOnce() + 'a       // res=Param, ident='a, source=OutlivesBound, syntax=ExplicitBound
 /// {
 ///     || {}
 /// }
 ///
 /// fn capture_precise<'a>() ->
-///     impl FnOnce() + use<'a>  // res=Param, ident='a, source=PreciseCapturing, syntax=Named
+///     impl FnOnce() + use<'a>  // res=Param, ident='a, source=PreciseCapturing, syntax=ExplicitBound
 /// {
 ///     || {}
 /// }
 ///
 /// // (commented out because these cases trigger errors)
-/// // struct S1<'a>(&'a str);   // res=Param, name='a, source=Reference, syntax=Named
-/// // struct S2(S1);            // res=Error, name='_, source=Path, syntax=Hidden
-/// // struct S3(S1<'_>);        // res=Error, name='_, source=Path, syntax=Anonymous
-/// // struct S4(S1<'a>);        // res=Error, name='a, source=Path, syntax=Named
+/// // struct S1<'a>(&'a str);   // res=Param, name='a, source=Reference, syntax=ExplicitBound
+/// // struct S2(S1);            // res=Error, name='_, source=Path, syntax=Implicit
+/// // struct S3(S1<'_>);        // res=Error, name='_, source=Path, syntax=ExplicitAnonymous
+/// // struct S4(S1<'a>);        // res=Error, name='a, source=Path, syntax=ExplicitBound
 /// ```
 ///
-/// Some combinations that cannot occur are `LifetimeSyntax::Hidden` with
+/// Some combinations that cannot occur are `LifetimeSyntax::Implicit` with
 /// `LifetimeSource::OutlivesBound` or `LifetimeSource::PreciseCapturing`
 /// — there's no way to "elide" these lifetimes.
 #[derive(Debug, Copy, Clone, HashStable_Generic)]
@@ -206,7 +206,7 @@ impl ParamName {
     }
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq, HashStable_Generic)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, HashStable_Generic)]
 pub enum LifetimeKind {
     /// User-given names or fresh (synthetic) names.
     Param(LocalDefId),
@@ -287,12 +287,8 @@ impl Lifetime {
         self.ident.name == kw::UnderscoreLifetime
     }
 
-    pub fn is_syntactically_hidden(&self) -> bool {
-        matches!(self.syntax, LifetimeSyntax::Hidden)
-    }
-
-    pub fn is_syntactically_anonymous(&self) -> bool {
-        matches!(self.syntax, LifetimeSyntax::Anonymous)
+    pub fn is_implicit(&self) -> bool {
+        matches!(self.syntax, LifetimeSyntax::Implicit)
     }
 
     pub fn is_static(&self) -> bool {
@@ -307,28 +303,28 @@ impl Lifetime {
 
         match (self.syntax, self.source) {
             // The user wrote `'a` or `'_`.
-            (Named | Anonymous, _) => (self.ident.span, format!("{new_lifetime}")),
+            (ExplicitBound | ExplicitAnonymous, _) => (self.ident.span, format!("{new_lifetime}")),
 
             // The user wrote `Path<T>`, and omitted the `'_,`.
-            (Hidden, Path { angle_brackets: AngleBrackets::Full }) => {
+            (Implicit, Path { angle_brackets: AngleBrackets::Full }) => {
                 (self.ident.span, format!("{new_lifetime}, "))
             }
 
             // The user wrote `Path<>`, and omitted the `'_`..
-            (Hidden, Path { angle_brackets: AngleBrackets::Empty }) => {
+            (Implicit, Path { angle_brackets: AngleBrackets::Empty }) => {
                 (self.ident.span, format!("{new_lifetime}"))
             }
 
             // The user wrote `Path` and omitted the `<'_>`.
-            (Hidden, Path { angle_brackets: AngleBrackets::Missing }) => {
+            (Implicit, Path { angle_brackets: AngleBrackets::Missing }) => {
                 (self.ident.span.shrink_to_hi(), format!("<{new_lifetime}>"))
             }
 
             // The user wrote `&type` or `&mut type`.
-            (Hidden, Reference) => (self.ident.span, format!("{new_lifetime} ")),
+            (Implicit, Reference) => (self.ident.span, format!("{new_lifetime} ")),
 
-            (Hidden, source) => {
-                unreachable!("can't suggest for a hidden lifetime of {source:?}")
+            (Implicit, source) => {
+                unreachable!("can't suggest for a implicit lifetime of {source:?}")
             }
         }
     }
@@ -347,7 +343,7 @@ pub struct Path<'hir, R = Res> {
 }
 
 /// Up to three resolutions for type, value and macro namespaces.
-pub type UsePath<'hir> = Path<'hir, SmallVec<[Res; 3]>>;
+pub type UsePath<'hir> = Path<'hir, PerNS<Option<Res>>>;
 
 impl Path<'_> {
     pub fn is_global(&self) -> bool {
@@ -2061,12 +2057,19 @@ impl CoroutineKind {
             CoroutineKind::Coroutine(mov) => mov,
         }
     }
-}
 
-impl CoroutineKind {
     pub fn is_fn_like(self) -> bool {
         matches!(self, CoroutineKind::Desugared(_, CoroutineSource::Fn))
     }
+
+    pub fn to_plural_string(&self) -> String {
+        match self {
+            CoroutineKind::Desugared(d, CoroutineSource::Fn) => format!("{d:#}fn bodies"),
+            CoroutineKind::Desugared(d, CoroutineSource::Block) => format!("{d:#}blocks"),
+            CoroutineKind::Desugared(d, CoroutineSource::Closure) => format!("{d:#}closure bodies"),
+            CoroutineKind::Coroutine(_) => "coroutines".to_string(),
+        }
+    }
 }
 
 impl fmt::Display for CoroutineKind {
@@ -2370,6 +2373,10 @@ impl Expr<'_> {
             // Lang item paths cannot currently be local variables or statics.
             ExprKind::Path(QPath::LangItem(..)) => false,
 
+            // Suppress errors for bad expressions.
+            ExprKind::Err(_guar)
+            | ExprKind::Let(&LetExpr { recovered: ast::Recovered::Yes(_guar), .. }) => true,
+
             // Partially qualified paths in expressions can only legally
             // refer to associated items which are always rvalues.
             ExprKind::Path(QPath::TypeRelative(..))
@@ -2401,8 +2408,7 @@ impl Expr<'_> {
             | ExprKind::Binary(..)
             | ExprKind::Yield(..)
             | ExprKind::Cast(..)
-            | ExprKind::DropTemps(..)
-            | ExprKind::Err(_) => false,
+            | ExprKind::DropTemps(..) => false,
         }
     }
 
@@ -4106,11 +4112,11 @@ impl<'hir> Item<'hir> {
 
         expect_use, (&'hir UsePath<'hir>, UseKind), ItemKind::Use(p, uk), (p, *uk);
 
-        expect_static, (Ident, &'hir Ty<'hir>, Mutability, BodyId),
-            ItemKind::Static(ident, ty, mutbl, body), (*ident, ty, *mutbl, *body);
+        expect_static, (Mutability, Ident, &'hir Ty<'hir>, BodyId),
+            ItemKind::Static(mutbl, ident, ty, body), (*mutbl, *ident, ty, *body);
 
-        expect_const, (Ident, &'hir Ty<'hir>, &'hir Generics<'hir>, BodyId),
-            ItemKind::Const(ident, ty, generics, body), (*ident, ty, generics, *body);
+        expect_const, (Ident, &'hir Generics<'hir>, &'hir Ty<'hir>, BodyId),
+            ItemKind::Const(ident, generics, ty, body), (*ident, generics, ty, *body);
 
         expect_fn, (Ident, &FnSig<'hir>, &'hir Generics<'hir>, BodyId),
             ItemKind::Fn { ident, sig, generics, body, .. }, (*ident, sig, generics, *body);
@@ -4125,17 +4131,17 @@ impl<'hir> Item<'hir> {
 
         expect_global_asm, &'hir InlineAsm<'hir>, ItemKind::GlobalAsm { asm, .. }, asm;
 
-        expect_ty_alias, (Ident, &'hir Ty<'hir>, &'hir Generics<'hir>),
-            ItemKind::TyAlias(ident, ty, generics), (*ident, ty, generics);
+        expect_ty_alias, (Ident, &'hir Generics<'hir>, &'hir Ty<'hir>),
+            ItemKind::TyAlias(ident, generics, ty), (*ident, generics, ty);
 
-        expect_enum, (Ident, &EnumDef<'hir>, &'hir Generics<'hir>),
-            ItemKind::Enum(ident, def, generics), (*ident, def, generics);
+        expect_enum, (Ident, &'hir Generics<'hir>, &EnumDef<'hir>),
+            ItemKind::Enum(ident, generics, def), (*ident, generics, def);
 
-        expect_struct, (Ident, &VariantData<'hir>, &'hir Generics<'hir>),
-            ItemKind::Struct(ident, data, generics), (*ident, data, generics);
+        expect_struct, (Ident, &'hir Generics<'hir>, &VariantData<'hir>),
+            ItemKind::Struct(ident, generics, data), (*ident, generics, data);
 
-        expect_union, (Ident, &VariantData<'hir>, &'hir Generics<'hir>),
-            ItemKind::Union(ident, data, generics), (*ident, data, generics);
+        expect_union, (Ident, &'hir Generics<'hir>, &VariantData<'hir>),
+            ItemKind::Union(ident, generics, data), (*ident, generics, data);
 
         expect_trait,
             (
@@ -4278,13 +4284,13 @@ pub enum ItemKind<'hir> {
     Use(&'hir UsePath<'hir>, UseKind),
 
     /// A `static` item.
-    Static(Ident, &'hir Ty<'hir>, Mutability, BodyId),
+    Static(Mutability, Ident, &'hir Ty<'hir>, BodyId),
     /// A `const` item.
-    Const(Ident, &'hir Ty<'hir>, &'hir Generics<'hir>, BodyId),
+    Const(Ident, &'hir Generics<'hir>, &'hir Ty<'hir>, BodyId),
     /// A function declaration.
     Fn {
-        ident: Ident,
         sig: FnSig<'hir>,
+        ident: Ident,
         generics: &'hir Generics<'hir>,
         body: BodyId,
         /// Whether this function actually has a body.
@@ -4309,13 +4315,13 @@ pub enum ItemKind<'hir> {
         fake_body: BodyId,
     },
     /// A type alias, e.g., `type Foo = Bar<u8>`.
-    TyAlias(Ident, &'hir Ty<'hir>, &'hir Generics<'hir>),
+    TyAlias(Ident, &'hir Generics<'hir>, &'hir Ty<'hir>),
     /// An enum definition, e.g., `enum Foo<A, B> { C<A>, D<B> }`.
-    Enum(Ident, EnumDef<'hir>, &'hir Generics<'hir>),
+    Enum(Ident, &'hir Generics<'hir>, EnumDef<'hir>),
     /// A struct definition, e.g., `struct Foo<A> {x: A}`.
-    Struct(Ident, VariantData<'hir>, &'hir Generics<'hir>),
+    Struct(Ident, &'hir Generics<'hir>, VariantData<'hir>),
     /// A union definition, e.g., `union Foo<A, B> {x: A, y: B}`.
-    Union(Ident, VariantData<'hir>, &'hir Generics<'hir>),
+    Union(Ident, &'hir Generics<'hir>, VariantData<'hir>),
     /// A trait definition.
     Trait(IsAuto, Safety, Ident, &'hir Generics<'hir>, GenericBounds<'hir>, &'hir [TraitItemRef]),
     /// A trait alias.
@@ -4352,7 +4358,7 @@ impl ItemKind<'_> {
         match *self {
             ItemKind::ExternCrate(_, ident)
             | ItemKind::Use(_, UseKind::Single(ident))
-            | ItemKind::Static(ident, ..)
+            | ItemKind::Static(_, ident, ..)
             | ItemKind::Const(ident, ..)
             | ItemKind::Fn { ident, .. }
             | ItemKind::Macro(ident, ..)
@@ -4374,11 +4380,11 @@ impl ItemKind<'_> {
     pub fn generics(&self) -> Option<&Generics<'_>> {
         Some(match self {
             ItemKind::Fn { generics, .. }
-            | ItemKind::TyAlias(_, _, generics)
-            | ItemKind::Const(_, _, generics, _)
-            | ItemKind::Enum(_, _, generics)
-            | ItemKind::Struct(_, _, generics)
-            | ItemKind::Union(_, _, generics)
+            | ItemKind::TyAlias(_, generics, _)
+            | ItemKind::Const(_, generics, _, _)
+            | ItemKind::Enum(_, generics, _)
+            | ItemKind::Struct(_, generics, _)
+            | ItemKind::Union(_, generics, _)
             | ItemKind::Trait(_, _, _, generics, _, _)
             | ItemKind::TraitAlias(_, generics, _)
             | ItemKind::Impl(Impl { generics, .. }) => generics,
@@ -4802,9 +4808,9 @@ impl<'hir> Node<'hir> {
     pub fn ty(self) -> Option<&'hir Ty<'hir>> {
         match self {
             Node::Item(it) => match it.kind {
-                ItemKind::TyAlias(_, ty, _)
-                | ItemKind::Static(_, ty, _, _)
-                | ItemKind::Const(_, ty, _, _) => Some(ty),
+                ItemKind::TyAlias(_, _, ty)
+                | ItemKind::Static(_, _, ty, _)
+                | ItemKind::Const(_, _, ty, _) => Some(ty),
                 ItemKind::Impl(impl_item) => Some(&impl_item.self_ty),
                 _ => None,
             },
@@ -4824,7 +4830,7 @@ impl<'hir> Node<'hir> {
 
     pub fn alias_ty(self) -> Option<&'hir Ty<'hir>> {
         match self {
-            Node::Item(Item { kind: ItemKind::TyAlias(_, ty, _), .. }) => Some(ty),
+            Node::Item(Item { kind: ItemKind::TyAlias(_, _, ty), .. }) => Some(ty),
             _ => None,
         }
     }
diff --git a/compiler/rustc_hir/src/hir/tests.rs b/compiler/rustc_hir/src/hir/tests.rs
index 8684adee29c..1fd793bc161 100644
--- a/compiler/rustc_hir/src/hir/tests.rs
+++ b/compiler/rustc_hir/src/hir/tests.rs
@@ -55,7 +55,7 @@ fn trait_object_roundtrips_impl(syntax: TraitObjectSyntax) {
         ident: Ident::new(sym::name, DUMMY_SP),
         kind: LifetimeKind::Static,
         source: LifetimeSource::Other,
-        syntax: LifetimeSyntax::Hidden,
+        syntax: LifetimeSyntax::Implicit,
     };
     let unambig = TyKind::TraitObject::<'_, ()>(&[], TaggedRef::new(&lt, syntax));
     let unambig_to_ambig = unsafe { std::mem::transmute::<_, TyKind<'_, AmbigArg>>(unambig) };
diff --git a/compiler/rustc_hir/src/intravisit.rs b/compiler/rustc_hir/src/intravisit.rs
index a60de4b1fc3..bebac3a4b78 100644
--- a/compiler/rustc_hir/src/intravisit.rs
+++ b/compiler/rustc_hir/src/intravisit.rs
@@ -200,6 +200,10 @@ use nested_filter::NestedFilter;
 /// explicitly, you need to override each method. (And you also need
 /// to monitor future changes to `Visitor` in case a new method with a
 /// new default implementation gets introduced.)
+///
+/// Every `walk_*` method uses deconstruction to access fields of structs and
+/// enums. This will result in a compile error if a field is added, which makes
+/// it more likely the appropriate visit call will be added for it.
 pub trait Visitor<'v>: Sized {
     // This type should not be overridden, it exists for convenient usage as `Self::MaybeTyCtxt`.
     type MaybeTyCtxt: HirTyCtxt<'v> = <Self::NestedFilter as NestedFilter<'v>>::MaybeTyCtxt;
@@ -527,13 +531,15 @@ pub trait VisitorExt<'v>: Visitor<'v> {
 impl<'v, V: Visitor<'v>> VisitorExt<'v> for V {}
 
 pub fn walk_param<'v, V: Visitor<'v>>(visitor: &mut V, param: &'v Param<'v>) -> V::Result {
-    try_visit!(visitor.visit_id(param.hir_id));
-    visitor.visit_pat(param.pat)
+    let Param { hir_id, pat, ty_span: _, span: _ } = param;
+    try_visit!(visitor.visit_id(*hir_id));
+    visitor.visit_pat(pat)
 }
 
 pub fn walk_item<'v, V: Visitor<'v>>(visitor: &mut V, item: &'v Item<'v>) -> V::Result {
+    let Item { owner_id: _, kind, span: _, vis_span: _ } = item;
     try_visit!(visitor.visit_id(item.hir_id()));
-    match item.kind {
+    match *kind {
         ItemKind::ExternCrate(orig_name, ident) => {
             visit_opt!(visitor, visit_name, orig_name);
             try_visit!(visitor.visit_ident(ident));
@@ -545,15 +551,15 @@ pub fn walk_item<'v, V: Visitor<'v>>(visitor: &mut V, item: &'v Item<'v>) -> V::
                 UseKind::Glob | UseKind::ListStem => {}
             }
         }
-        ItemKind::Static(ident, ref typ, _, body) => {
+        ItemKind::Static(_, ident, ref typ, body) => {
             try_visit!(visitor.visit_ident(ident));
             try_visit!(visitor.visit_ty_unambig(typ));
             try_visit!(visitor.visit_nested_body(body));
         }
-        ItemKind::Const(ident, ref typ, ref generics, body) => {
+        ItemKind::Const(ident, ref generics, ref typ, body) => {
             try_visit!(visitor.visit_ident(ident));
-            try_visit!(visitor.visit_ty_unambig(typ));
             try_visit!(visitor.visit_generics(generics));
+            try_visit!(visitor.visit_ty_unambig(typ));
             try_visit!(visitor.visit_nested_body(body));
         }
         ItemKind::Fn { ident, sig, generics, body: body_id, .. } => {
@@ -583,12 +589,12 @@ pub fn walk_item<'v, V: Visitor<'v>>(visitor: &mut V, item: &'v Item<'v>) -> V::
             // typeck results set correctly.
             try_visit!(visitor.visit_nested_body(fake_body));
         }
-        ItemKind::TyAlias(ident, ref ty, ref generics) => {
+        ItemKind::TyAlias(ident, ref generics, ref ty) => {
             try_visit!(visitor.visit_ident(ident));
-            try_visit!(visitor.visit_ty_unambig(ty));
             try_visit!(visitor.visit_generics(generics));
+            try_visit!(visitor.visit_ty_unambig(ty));
         }
-        ItemKind::Enum(ident, ref enum_definition, ref generics) => {
+        ItemKind::Enum(ident, ref generics, ref enum_definition) => {
             try_visit!(visitor.visit_ident(ident));
             try_visit!(visitor.visit_generics(generics));
             try_visit!(visitor.visit_enum_def(enum_definition));
@@ -609,8 +615,8 @@ pub fn walk_item<'v, V: Visitor<'v>>(visitor: &mut V, item: &'v Item<'v>) -> V::
             try_visit!(visitor.visit_ty_unambig(self_ty));
             walk_list!(visitor, visit_impl_item_ref, *items);
         }
-        ItemKind::Struct(ident, ref struct_definition, ref generics)
-        | ItemKind::Union(ident, ref struct_definition, ref generics) => {
+        ItemKind::Struct(ident, ref generics, ref struct_definition)
+        | ItemKind::Union(ident, ref generics, ref struct_definition) => {
             try_visit!(visitor.visit_ident(ident));
             try_visit!(visitor.visit_generics(generics));
             try_visit!(visitor.visit_variant_data(struct_definition));
@@ -631,8 +637,9 @@ pub fn walk_item<'v, V: Visitor<'v>>(visitor: &mut V, item: &'v Item<'v>) -> V::
 }
 
 pub fn walk_body<'v, V: Visitor<'v>>(visitor: &mut V, body: &Body<'v>) -> V::Result {
-    walk_list!(visitor, visit_param, body.params);
-    visitor.visit_expr(body.value)
+    let Body { params, value } = body;
+    walk_list!(visitor, visit_param, *params);
+    visitor.visit_expr(*value)
 }
 
 pub fn walk_ident<'v, V: Visitor<'v>>(visitor: &mut V, ident: Ident) -> V::Result {
@@ -640,7 +647,8 @@ pub fn walk_ident<'v, V: Visitor<'v>>(visitor: &mut V, ident: Ident) -> V::Resul
 }
 
 pub fn walk_mod<'v, V: Visitor<'v>>(visitor: &mut V, module: &'v Mod<'v>) -> V::Result {
-    walk_list!(visitor, visit_nested_item, module.item_ids.iter().copied());
+    let Mod { spans: _, item_ids } = module;
+    walk_list!(visitor, visit_nested_item, item_ids.iter().copied());
     V::Result::output()
 }
 
@@ -648,10 +656,11 @@ pub fn walk_foreign_item<'v, V: Visitor<'v>>(
     visitor: &mut V,
     foreign_item: &'v ForeignItem<'v>,
 ) -> V::Result {
+    let ForeignItem { ident, kind, owner_id: _, span: _, vis_span: _ } = foreign_item;
     try_visit!(visitor.visit_id(foreign_item.hir_id()));
-    try_visit!(visitor.visit_ident(foreign_item.ident));
+    try_visit!(visitor.visit_ident(*ident));
 
-    match foreign_item.kind {
+    match *kind {
         ForeignItemKind::Fn(ref sig, param_idents, ref generics) => {
             try_visit!(visitor.visit_generics(generics));
             try_visit!(visitor.visit_fn_decl(sig.decl));
@@ -670,24 +679,27 @@ pub fn walk_foreign_item<'v, V: Visitor<'v>>(
 pub fn walk_local<'v, V: Visitor<'v>>(visitor: &mut V, local: &'v LetStmt<'v>) -> V::Result {
     // Intentionally visiting the expr first - the initialization expr
     // dominates the local's definition.
-    visit_opt!(visitor, visit_expr, local.init);
-    try_visit!(visitor.visit_id(local.hir_id));
-    try_visit!(visitor.visit_pat(local.pat));
-    visit_opt!(visitor, visit_block, local.els);
-    visit_opt!(visitor, visit_ty_unambig, local.ty);
+    let LetStmt { super_: _, pat, ty, init, els, hir_id, span: _, source: _ } = local;
+    visit_opt!(visitor, visit_expr, *init);
+    try_visit!(visitor.visit_id(*hir_id));
+    try_visit!(visitor.visit_pat(*pat));
+    visit_opt!(visitor, visit_block, *els);
+    visit_opt!(visitor, visit_ty_unambig, *ty);
     V::Result::output()
 }
 
 pub fn walk_block<'v, V: Visitor<'v>>(visitor: &mut V, block: &'v Block<'v>) -> V::Result {
-    try_visit!(visitor.visit_id(block.hir_id));
-    walk_list!(visitor, visit_stmt, block.stmts);
-    visit_opt!(visitor, visit_expr, block.expr);
+    let Block { stmts, expr, hir_id, rules: _, span: _, targeted_by_break: _ } = block;
+    try_visit!(visitor.visit_id(*hir_id));
+    walk_list!(visitor, visit_stmt, *stmts);
+    visit_opt!(visitor, visit_expr, *expr);
     V::Result::output()
 }
 
 pub fn walk_stmt<'v, V: Visitor<'v>>(visitor: &mut V, statement: &'v Stmt<'v>) -> V::Result {
-    try_visit!(visitor.visit_id(statement.hir_id));
-    match statement.kind {
+    let Stmt { kind, hir_id, span: _ } = statement;
+    try_visit!(visitor.visit_id(*hir_id));
+    match *kind {
         StmtKind::Let(ref local) => visitor.visit_local(local),
         StmtKind::Item(item) => visitor.visit_nested_item(item),
         StmtKind::Expr(ref expression) | StmtKind::Semi(ref expression) => {
@@ -697,15 +709,17 @@ pub fn walk_stmt<'v, V: Visitor<'v>>(visitor: &mut V, statement: &'v Stmt<'v>) -
 }
 
 pub fn walk_arm<'v, V: Visitor<'v>>(visitor: &mut V, arm: &'v Arm<'v>) -> V::Result {
-    try_visit!(visitor.visit_id(arm.hir_id));
-    try_visit!(visitor.visit_pat(arm.pat));
-    visit_opt!(visitor, visit_expr, arm.guard);
-    visitor.visit_expr(arm.body)
+    let Arm { hir_id, span: _, pat, guard, body } = arm;
+    try_visit!(visitor.visit_id(*hir_id));
+    try_visit!(visitor.visit_pat(*pat));
+    visit_opt!(visitor, visit_expr, *guard);
+    visitor.visit_expr(*body)
 }
 
 pub fn walk_ty_pat<'v, V: Visitor<'v>>(visitor: &mut V, pattern: &'v TyPat<'v>) -> V::Result {
-    try_visit!(visitor.visit_id(pattern.hir_id));
-    match pattern.kind {
+    let TyPat { kind, hir_id, span: _ } = pattern;
+    try_visit!(visitor.visit_id(*hir_id));
+    match *kind {
         TyPatKind::Range(lower_bound, upper_bound) => {
             try_visit!(visitor.visit_const_arg_unambig(lower_bound));
             try_visit!(visitor.visit_const_arg_unambig(upper_bound));
@@ -717,14 +731,15 @@ pub fn walk_ty_pat<'v, V: Visitor<'v>>(visitor: &mut V, pattern: &'v TyPat<'v>)
 }
 
 pub fn walk_pat<'v, V: Visitor<'v>>(visitor: &mut V, pattern: &'v Pat<'v>) -> V::Result {
-    try_visit!(visitor.visit_id(pattern.hir_id));
-    match pattern.kind {
+    let Pat { hir_id, kind, span, default_binding_modes: _ } = pattern;
+    try_visit!(visitor.visit_id(*hir_id));
+    match *kind {
         PatKind::TupleStruct(ref qpath, children, _) => {
-            try_visit!(visitor.visit_qpath(qpath, pattern.hir_id, pattern.span));
+            try_visit!(visitor.visit_qpath(qpath, *hir_id, *span));
             walk_list!(visitor, visit_pat, children);
         }
         PatKind::Struct(ref qpath, fields, _) => {
-            try_visit!(visitor.visit_qpath(qpath, pattern.hir_id, pattern.span));
+            try_visit!(visitor.visit_qpath(qpath, *hir_id, *span));
             walk_list!(visitor, visit_pat_field, fields);
         }
         PatKind::Or(pats) => walk_list!(visitor, visit_pat, pats),
@@ -760,36 +775,41 @@ pub fn walk_pat<'v, V: Visitor<'v>>(visitor: &mut V, pattern: &'v Pat<'v>) -> V:
 }
 
 pub fn walk_pat_field<'v, V: Visitor<'v>>(visitor: &mut V, field: &'v PatField<'v>) -> V::Result {
-    try_visit!(visitor.visit_id(field.hir_id));
-    try_visit!(visitor.visit_ident(field.ident));
-    visitor.visit_pat(field.pat)
+    let PatField { hir_id, ident, pat, is_shorthand: _, span: _ } = field;
+    try_visit!(visitor.visit_id(*hir_id));
+    try_visit!(visitor.visit_ident(*ident));
+    visitor.visit_pat(*pat)
 }
 
 pub fn walk_pat_expr<'v, V: Visitor<'v>>(visitor: &mut V, expr: &'v PatExpr<'v>) -> V::Result {
-    try_visit!(visitor.visit_id(expr.hir_id));
-    match &expr.kind {
-        PatExprKind::Lit { lit, negated } => visitor.visit_lit(expr.hir_id, lit, *negated),
+    let PatExpr { hir_id, span, kind } = expr;
+    try_visit!(visitor.visit_id(*hir_id));
+    match kind {
+        PatExprKind::Lit { lit, negated } => visitor.visit_lit(*hir_id, lit, *negated),
         PatExprKind::ConstBlock(c) => visitor.visit_inline_const(c),
-        PatExprKind::Path(qpath) => visitor.visit_qpath(qpath, expr.hir_id, expr.span),
+        PatExprKind::Path(qpath) => visitor.visit_qpath(qpath, *hir_id, *span),
     }
 }
 
 pub fn walk_anon_const<'v, V: Visitor<'v>>(visitor: &mut V, constant: &'v AnonConst) -> V::Result {
-    try_visit!(visitor.visit_id(constant.hir_id));
-    visitor.visit_nested_body(constant.body)
+    let AnonConst { hir_id, def_id: _, body, span: _ } = constant;
+    try_visit!(visitor.visit_id(*hir_id));
+    visitor.visit_nested_body(*body)
 }
 
 pub fn walk_inline_const<'v, V: Visitor<'v>>(
     visitor: &mut V,
     constant: &'v ConstBlock,
 ) -> V::Result {
-    try_visit!(visitor.visit_id(constant.hir_id));
-    visitor.visit_nested_body(constant.body)
+    let ConstBlock { hir_id, def_id: _, body } = constant;
+    try_visit!(visitor.visit_id(*hir_id));
+    visitor.visit_nested_body(*body)
 }
 
 pub fn walk_expr<'v, V: Visitor<'v>>(visitor: &mut V, expression: &'v Expr<'v>) -> V::Result {
-    try_visit!(visitor.visit_id(expression.hir_id));
-    match expression.kind {
+    let Expr { hir_id, kind, span } = expression;
+    try_visit!(visitor.visit_id(*hir_id));
+    match *kind {
         ExprKind::Array(subexpressions) => {
             walk_list!(visitor, visit_expr, subexpressions);
         }
@@ -801,7 +821,7 @@ pub fn walk_expr<'v, V: Visitor<'v>>(visitor: &mut V, expression: &'v Expr<'v>)
             try_visit!(visitor.visit_const_arg_unambig(count));
         }
         ExprKind::Struct(ref qpath, fields, ref optional_base) => {
-            try_visit!(visitor.visit_qpath(qpath, expression.hir_id, expression.span));
+            try_visit!(visitor.visit_qpath(qpath, *hir_id, *span));
             walk_list!(visitor, visit_expr_field, fields);
             match optional_base {
                 StructTailExpr::Base(base) => try_visit!(visitor.visit_expr(base)),
@@ -869,7 +889,7 @@ pub fn walk_expr<'v, V: Visitor<'v>>(visitor: &mut V, expression: &'v Expr<'v>)
             constness: _,
         }) => {
             walk_list!(visitor, visit_generic_param, bound_generic_params);
-            try_visit!(visitor.visit_fn(FnKind::Closure, fn_decl, body, expression.span, def_id));
+            try_visit!(visitor.visit_fn(FnKind::Closure, fn_decl, body, *span, def_id));
         }
         ExprKind::Block(ref block, ref opt_label) => {
             visit_opt!(visitor, visit_label, opt_label);
@@ -892,7 +912,7 @@ pub fn walk_expr<'v, V: Visitor<'v>>(visitor: &mut V, expression: &'v Expr<'v>)
             try_visit!(visitor.visit_expr(index_expression));
         }
         ExprKind::Path(ref qpath) => {
-            try_visit!(visitor.visit_qpath(qpath, expression.hir_id, expression.span));
+            try_visit!(visitor.visit_qpath(qpath, *hir_id, *span));
         }
         ExprKind::Break(ref destination, ref opt_expr) => {
             visit_opt!(visitor, visit_label, &destination.label);
@@ -906,7 +926,7 @@ pub fn walk_expr<'v, V: Visitor<'v>>(visitor: &mut V, expression: &'v Expr<'v>)
         }
         ExprKind::Become(ref expr) => try_visit!(visitor.visit_expr(expr)),
         ExprKind::InlineAsm(ref asm) => {
-            try_visit!(visitor.visit_inline_asm(asm, expression.hir_id));
+            try_visit!(visitor.visit_inline_asm(asm, *hir_id));
         }
         ExprKind::OffsetOf(ref container, ref fields) => {
             try_visit!(visitor.visit_ty_unambig(container));
@@ -919,16 +939,17 @@ pub fn walk_expr<'v, V: Visitor<'v>>(visitor: &mut V, expression: &'v Expr<'v>)
             try_visit!(visitor.visit_expr(expr));
             visit_opt!(visitor, visit_ty_unambig, ty);
         }
-        ExprKind::Lit(lit) => try_visit!(visitor.visit_lit(expression.hir_id, lit, false)),
+        ExprKind::Lit(lit) => try_visit!(visitor.visit_lit(*hir_id, lit, false)),
         ExprKind::Err(_) => {}
     }
     V::Result::output()
 }
 
 pub fn walk_expr_field<'v, V: Visitor<'v>>(visitor: &mut V, field: &'v ExprField<'v>) -> V::Result {
-    try_visit!(visitor.visit_id(field.hir_id));
-    try_visit!(visitor.visit_ident(field.ident));
-    visitor.visit_expr(field.expr)
+    let ExprField { hir_id, ident, expr, span: _, is_shorthand: _ } = field;
+    try_visit!(visitor.visit_id(*hir_id));
+    try_visit!(visitor.visit_ident(*ident));
+    visitor.visit_expr(*expr)
 }
 /// We track whether an infer var is from a [`Ty`], [`ConstArg`], or [`GenericArg`] so that
 /// HIR visitors overriding [`Visitor::visit_infer`] can determine what kind of infer is being visited
@@ -946,7 +967,10 @@ pub fn walk_generic_arg<'v, V: Visitor<'v>>(
         GenericArg::Lifetime(lt) => visitor.visit_lifetime(lt),
         GenericArg::Type(ty) => visitor.visit_ty(ty),
         GenericArg::Const(ct) => visitor.visit_const_arg(ct),
-        GenericArg::Infer(inf) => visitor.visit_infer(inf.hir_id, inf.span, InferKind::Ambig(inf)),
+        GenericArg::Infer(inf) => {
+            let InferArg { hir_id, span } = inf;
+            visitor.visit_infer(*hir_id, *span, InferKind::Ambig(inf))
+        }
     }
 }
 
@@ -954,16 +978,18 @@ pub fn walk_unambig_ty<'v, V: Visitor<'v>>(visitor: &mut V, typ: &'v Ty<'v>) ->
     match typ.try_as_ambig_ty() {
         Some(ambig_ty) => visitor.visit_ty(ambig_ty),
         None => {
-            try_visit!(visitor.visit_id(typ.hir_id));
-            visitor.visit_infer(typ.hir_id, typ.span, InferKind::Ty(typ))
+            let Ty { hir_id, span, kind: _ } = typ;
+            try_visit!(visitor.visit_id(*hir_id));
+            visitor.visit_infer(*hir_id, *span, InferKind::Ty(typ))
         }
     }
 }
 
 pub fn walk_ty<'v, V: Visitor<'v>>(visitor: &mut V, typ: &'v Ty<'v, AmbigArg>) -> V::Result {
-    try_visit!(visitor.visit_id(typ.hir_id));
+    let Ty { hir_id, span: _, kind } = typ;
+    try_visit!(visitor.visit_id(*hir_id));
 
-    match typ.kind {
+    match *kind {
         TyKind::Slice(ref ty) => try_visit!(visitor.visit_ty_unambig(ty)),
         TyKind::Ptr(ref mutable_type) => try_visit!(visitor.visit_ty_unambig(mutable_type.ty)),
         TyKind::Ref(ref lifetime, ref mutable_type) => {
@@ -1018,8 +1044,9 @@ pub fn walk_const_arg<'v, V: Visitor<'v>>(
     match const_arg.try_as_ambig_ct() {
         Some(ambig_ct) => visitor.visit_const_arg(ambig_ct),
         None => {
-            try_visit!(visitor.visit_id(const_arg.hir_id));
-            visitor.visit_infer(const_arg.hir_id, const_arg.span(), InferKind::Const(const_arg))
+            let ConstArg { hir_id, kind: _ } = const_arg;
+            try_visit!(visitor.visit_id(*hir_id));
+            visitor.visit_infer(*hir_id, const_arg.span(), InferKind::Const(const_arg))
         }
     }
 }
@@ -1028,9 +1055,10 @@ pub fn walk_ambig_const_arg<'v, V: Visitor<'v>>(
     visitor: &mut V,
     const_arg: &'v ConstArg<'v, AmbigArg>,
 ) -> V::Result {
-    try_visit!(visitor.visit_id(const_arg.hir_id));
-    match &const_arg.kind {
-        ConstArgKind::Path(qpath) => visitor.visit_qpath(qpath, const_arg.hir_id, qpath.span()),
+    let ConstArg { hir_id, kind } = const_arg;
+    try_visit!(visitor.visit_id(*hir_id));
+    match kind {
+        ConstArgKind::Path(qpath) => visitor.visit_qpath(qpath, *hir_id, qpath.span()),
         ConstArgKind::Anon(anon) => visitor.visit_anon_const(*anon),
     }
 }
@@ -1039,12 +1067,22 @@ pub fn walk_generic_param<'v, V: Visitor<'v>>(
     visitor: &mut V,
     param: &'v GenericParam<'v>,
 ) -> V::Result {
-    try_visit!(visitor.visit_id(param.hir_id));
-    match param.name {
+    let GenericParam {
+        hir_id,
+        def_id: _,
+        name,
+        span: _,
+        pure_wrt_drop: _,
+        kind,
+        colon_span: _,
+        source: _,
+    } = param;
+    try_visit!(visitor.visit_id(*hir_id));
+    match *name {
         ParamName::Plain(ident) | ParamName::Error(ident) => try_visit!(visitor.visit_ident(ident)),
         ParamName::Fresh => {}
     }
-    match param.kind {
+    match *kind {
         GenericParamKind::Lifetime { .. } => {}
         GenericParamKind::Type { ref default, .. } => {
             visit_opt!(visitor, visit_ty_unambig, default)
@@ -1052,7 +1090,7 @@ pub fn walk_generic_param<'v, V: Visitor<'v>>(
         GenericParamKind::Const { ref ty, ref default, synthetic: _ } => {
             try_visit!(visitor.visit_ty_unambig(ty));
             if let Some(default) = default {
-                try_visit!(visitor.visit_const_param_default(param.hir_id, default));
+                try_visit!(visitor.visit_const_param_default(*hir_id, default));
             }
         }
     }
@@ -1067,8 +1105,15 @@ pub fn walk_const_param_default<'v, V: Visitor<'v>>(
 }
 
 pub fn walk_generics<'v, V: Visitor<'v>>(visitor: &mut V, generics: &'v Generics<'v>) -> V::Result {
-    walk_list!(visitor, visit_generic_param, generics.params);
-    walk_list!(visitor, visit_where_predicate, generics.predicates);
+    let &Generics {
+        params,
+        predicates,
+        has_where_clause_predicates: _,
+        where_clause_span: _,
+        span: _,
+    } = generics;
+    walk_list!(visitor, visit_generic_param, params);
+    walk_list!(visitor, visit_where_predicate, predicates);
     V::Result::output()
 }
 
@@ -1109,8 +1154,10 @@ pub fn walk_fn_decl<'v, V: Visitor<'v>>(
     visitor: &mut V,
     function_declaration: &'v FnDecl<'v>,
 ) -> V::Result {
-    walk_list!(visitor, visit_ty_unambig, function_declaration.inputs);
-    visitor.visit_fn_ret_ty(&function_declaration.output)
+    let FnDecl { inputs, output, c_variadic: _, implicit_self: _, lifetime_elision_allowed: _ } =
+        function_declaration;
+    walk_list!(visitor, visit_ty_unambig, *inputs);
+    visitor.visit_fn_ret_ty(output)
 }
 
 pub fn walk_fn_ret_ty<'v, V: Visitor<'v>>(visitor: &mut V, ret_ty: &'v FnRetTy<'v>) -> V::Result {
@@ -1148,7 +1195,7 @@ pub fn walk_use<'v, V: Visitor<'v>>(
     hir_id: HirId,
 ) -> V::Result {
     let UsePath { segments, ref res, span } = *path;
-    for &res in res {
+    for res in res.present_items() {
         try_visit!(visitor.visit_path(&Path { segments, res, span }, hir_id));
     }
     V::Result::output()
@@ -1158,7 +1205,6 @@ pub fn walk_trait_item<'v, V: Visitor<'v>>(
     visitor: &mut V,
     trait_item: &'v TraitItem<'v>,
 ) -> V::Result {
-    // N.B., deliberately force a compilation error if/when new fields are added.
     let TraitItem { ident, generics, ref defaultness, ref kind, span, owner_id: _ } = *trait_item;
     let hir_id = trait_item.hir_id();
     try_visit!(visitor.visit_ident(ident));
@@ -1197,7 +1243,6 @@ pub fn walk_trait_item_ref<'v, V: Visitor<'v>>(
     visitor: &mut V,
     trait_item_ref: &'v TraitItemRef,
 ) -> V::Result {
-    // N.B., deliberately force a compilation error if/when new fields are added.
     let TraitItemRef { id, ident, ref kind, span: _ } = *trait_item_ref;
     try_visit!(visitor.visit_nested_trait_item(id));
     try_visit!(visitor.visit_ident(ident));
@@ -1208,7 +1253,6 @@ pub fn walk_impl_item<'v, V: Visitor<'v>>(
     visitor: &mut V,
     impl_item: &'v ImplItem<'v>,
 ) -> V::Result {
-    // N.B., deliberately force a compilation error if/when new fields are added.
     let ImplItem {
         owner_id: _,
         ident,
@@ -1243,7 +1287,6 @@ pub fn walk_foreign_item_ref<'v, V: Visitor<'v>>(
     visitor: &mut V,
     foreign_item_ref: &'v ForeignItemRef,
 ) -> V::Result {
-    // N.B., deliberately force a compilation error if/when new fields are added.
     let ForeignItemRef { id, ident, span: _ } = *foreign_item_ref;
     try_visit!(visitor.visit_nested_foreign_item(id));
     visitor.visit_ident(ident)
@@ -1253,7 +1296,6 @@ pub fn walk_impl_item_ref<'v, V: Visitor<'v>>(
     visitor: &mut V,
     impl_item_ref: &'v ImplItemRef,
 ) -> V::Result {
-    // N.B., deliberately force a compilation error if/when new fields are added.
     let ImplItemRef { id, ident, ref kind, span: _, trait_item_def_id: _ } = *impl_item_ref;
     try_visit!(visitor.visit_nested_impl_item(id));
     try_visit!(visitor.visit_ident(ident));
@@ -1264,8 +1306,9 @@ pub fn walk_trait_ref<'v, V: Visitor<'v>>(
     visitor: &mut V,
     trait_ref: &'v TraitRef<'v>,
 ) -> V::Result {
-    try_visit!(visitor.visit_id(trait_ref.hir_ref_id));
-    visitor.visit_path(trait_ref.path, trait_ref.hir_ref_id)
+    let TraitRef { hir_ref_id, path } = trait_ref;
+    try_visit!(visitor.visit_id(*hir_ref_id));
+    visitor.visit_path(*path, *hir_ref_id)
 }
 
 pub fn walk_param_bound<'v, V: Visitor<'v>>(
@@ -1288,7 +1331,11 @@ pub fn walk_precise_capturing_arg<'v, V: Visitor<'v>>(
 ) -> V::Result {
     match *arg {
         PreciseCapturingArg::Lifetime(lt) => visitor.visit_lifetime(lt),
-        PreciseCapturingArg::Param(param) => visitor.visit_id(param.hir_id),
+        PreciseCapturingArg::Param(param) => {
+            let PreciseCapturingNonLifetimeArg { hir_id, ident, res: _ } = param;
+            try_visit!(visitor.visit_id(hir_id));
+            visitor.visit_ident(ident)
+        }
     }
 }
 
@@ -1296,8 +1343,9 @@ pub fn walk_poly_trait_ref<'v, V: Visitor<'v>>(
     visitor: &mut V,
     trait_ref: &'v PolyTraitRef<'v>,
 ) -> V::Result {
-    walk_list!(visitor, visit_generic_param, trait_ref.bound_generic_params);
-    visitor.visit_trait_ref(&trait_ref.trait_ref)
+    let PolyTraitRef { bound_generic_params, modifiers: _, trait_ref, span: _ } = trait_ref;
+    walk_list!(visitor, visit_generic_param, *bound_generic_params);
+    visitor.visit_trait_ref(trait_ref)
 }
 
 pub fn walk_opaque_ty<'v, V: Visitor<'v>>(visitor: &mut V, opaque: &'v OpaqueTy<'v>) -> V::Result {
@@ -1330,29 +1378,34 @@ pub fn walk_enum_def<'v, V: Visitor<'v>>(
     visitor: &mut V,
     enum_definition: &'v EnumDef<'v>,
 ) -> V::Result {
-    walk_list!(visitor, visit_variant, enum_definition.variants);
+    let EnumDef { variants } = enum_definition;
+    walk_list!(visitor, visit_variant, *variants);
     V::Result::output()
 }
 
 pub fn walk_variant<'v, V: Visitor<'v>>(visitor: &mut V, variant: &'v Variant<'v>) -> V::Result {
-    try_visit!(visitor.visit_ident(variant.ident));
-    try_visit!(visitor.visit_id(variant.hir_id));
-    try_visit!(visitor.visit_variant_data(&variant.data));
-    visit_opt!(visitor, visit_anon_const, &variant.disr_expr);
+    let Variant { ident, hir_id, def_id: _, data, disr_expr, span: _ } = variant;
+    try_visit!(visitor.visit_ident(*ident));
+    try_visit!(visitor.visit_id(*hir_id));
+    try_visit!(visitor.visit_variant_data(data));
+    visit_opt!(visitor, visit_anon_const, disr_expr);
     V::Result::output()
 }
 
 pub fn walk_label<'v, V: Visitor<'v>>(visitor: &mut V, label: &'v Label) -> V::Result {
-    visitor.visit_ident(label.ident)
+    let Label { ident } = label;
+    visitor.visit_ident(*ident)
 }
 
 pub fn walk_inf<'v, V: Visitor<'v>>(visitor: &mut V, inf: &'v InferArg) -> V::Result {
-    visitor.visit_id(inf.hir_id)
+    let InferArg { hir_id, span: _ } = inf;
+    visitor.visit_id(*hir_id)
 }
 
 pub fn walk_lifetime<'v, V: Visitor<'v>>(visitor: &mut V, lifetime: &'v Lifetime) -> V::Result {
-    try_visit!(visitor.visit_id(lifetime.hir_id));
-    visitor.visit_ident(lifetime.ident)
+    let Lifetime { hir_id, ident, kind: _, source: _, syntax: _ } = lifetime;
+    try_visit!(visitor.visit_id(*hir_id));
+    visitor.visit_ident(*ident)
 }
 
 pub fn walk_qpath<'v, V: Visitor<'v>>(
@@ -1374,7 +1427,8 @@ pub fn walk_qpath<'v, V: Visitor<'v>>(
 }
 
 pub fn walk_path<'v, V: Visitor<'v>>(visitor: &mut V, path: &Path<'v>) -> V::Result {
-    walk_list!(visitor, visit_path_segment, path.segments);
+    let Path { segments, span: _, res: _ } = path;
+    walk_list!(visitor, visit_path_segment, *segments);
     V::Result::output()
 }
 
@@ -1382,9 +1436,10 @@ pub fn walk_path_segment<'v, V: Visitor<'v>>(
     visitor: &mut V,
     segment: &'v PathSegment<'v>,
 ) -> V::Result {
-    try_visit!(visitor.visit_ident(segment.ident));
-    try_visit!(visitor.visit_id(segment.hir_id));
-    visit_opt!(visitor, visit_generic_args, segment.args);
+    let PathSegment { ident, hir_id, res: _, args, infer_args: _ } = segment;
+    try_visit!(visitor.visit_ident(*ident));
+    try_visit!(visitor.visit_id(*hir_id));
+    visit_opt!(visitor, visit_generic_args, *args);
     V::Result::output()
 }
 
@@ -1392,8 +1447,9 @@ pub fn walk_generic_args<'v, V: Visitor<'v>>(
     visitor: &mut V,
     generic_args: &'v GenericArgs<'v>,
 ) -> V::Result {
-    walk_list!(visitor, visit_generic_arg, generic_args.args);
-    walk_list!(visitor, visit_assoc_item_constraint, generic_args.constraints);
+    let GenericArgs { args, constraints, parenthesized: _, span_ext: _ } = generic_args;
+    walk_list!(visitor, visit_generic_arg, *args);
+    walk_list!(visitor, visit_assoc_item_constraint, *constraints);
     V::Result::output()
 }
 
@@ -1401,9 +1457,10 @@ pub fn walk_assoc_item_constraint<'v, V: Visitor<'v>>(
     visitor: &mut V,
     constraint: &'v AssocItemConstraint<'v>,
 ) -> V::Result {
-    try_visit!(visitor.visit_id(constraint.hir_id));
-    try_visit!(visitor.visit_ident(constraint.ident));
-    try_visit!(visitor.visit_generic_args(constraint.gen_args));
+    let AssocItemConstraint { hir_id, ident, gen_args, kind: _, span: _ } = constraint;
+    try_visit!(visitor.visit_id(*hir_id));
+    try_visit!(visitor.visit_ident(*ident));
+    try_visit!(visitor.visit_generic_args(*gen_args));
     match constraint.kind {
         AssocItemConstraintKind::Equality { ref term } => match term {
             Term::Ty(ty) => try_visit!(visitor.visit_ty_unambig(ty)),
diff --git a/compiler/rustc_hir/src/lib.rs b/compiler/rustc_hir/src/lib.rs
index 7a5ff890689..c6fe475b460 100644
--- a/compiler/rustc_hir/src/lib.rs
+++ b/compiler/rustc_hir/src/lib.rs
@@ -5,7 +5,6 @@
 // tidy-alphabetical-start
 #![allow(internal_features)]
 #![feature(associated_type_defaults)]
-#![feature(box_patterns)]
 #![feature(closure_track_caller)]
 #![feature(debug_closure_helpers)]
 #![feature(exhaustive_patterns)]
diff --git a/compiler/rustc_hir_analysis/Cargo.toml b/compiler/rustc_hir_analysis/Cargo.toml
index f2b82c679b9..899370b34e4 100644
--- a/compiler/rustc_hir_analysis/Cargo.toml
+++ b/compiler/rustc_hir_analysis/Cargo.toml
@@ -26,6 +26,7 @@ rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
+rustc_target = { path = "../rustc_target" }
 rustc_trait_selection = { path = "../rustc_trait_selection" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 tracing = "0.1"
diff --git a/compiler/rustc_hir_analysis/messages.ftl b/compiler/rustc_hir_analysis/messages.ftl
index f7f2b78f052..4fcd9f8a646 100644
--- a/compiler/rustc_hir_analysis/messages.ftl
+++ b/compiler/rustc_hir_analysis/messages.ftl
@@ -37,7 +37,7 @@ hir_analysis_assoc_kind_mismatch = expected {$expected}, found {$got}
 
 hir_analysis_assoc_kind_mismatch_wrap_in_braces_sugg = consider adding braces here
 
-hir_analysis_associated_type_trait_uninferred_generic_params = cannot use the associated {$what} of a trait with uninferred generic parameters
+hir_analysis_associated_type_trait_uninferred_generic_params = cannot use the {$what} of a trait with uninferred generic parameters
     .suggestion = use a fully qualified path with inferred lifetimes
 
 hir_analysis_associated_type_trait_uninferred_generic_params_multipart_suggestion = use a fully qualified path with explicit lifetimes
@@ -565,10 +565,6 @@ hir_analysis_unconstrained_generic_parameter = the {$param_def_kind} `{$param_na
 hir_analysis_unconstrained_opaque_type = unconstrained opaque type
     .note = `{$name}` must be used in combination with a concrete type within the same {$what}
 
-hir_analysis_unrecognized_atomic_operation =
-    unrecognized atomic operation function: `{$op}`
-    .label = unrecognized atomic operation
-
 hir_analysis_unrecognized_intrinsic_function =
     unrecognized intrinsic function: `{$name}`
     .label = unrecognized intrinsic
diff --git a/compiler/rustc_hir_analysis/src/check/check.rs b/compiler/rustc_hir_analysis/src/check/check.rs
index 06814eaefe8..064b42413f0 100644
--- a/compiler/rustc_hir_analysis/src/check/check.rs
+++ b/compiler/rustc_hir_analysis/src/check/check.rs
@@ -4,26 +4,28 @@ use std::ops::ControlFlow;
 use rustc_abi::FieldIdx;
 use rustc_attr_data_structures::ReprAttr::ReprPacked;
 use rustc_data_structures::unord::{UnordMap, UnordSet};
-use rustc_errors::MultiSpan;
 use rustc_errors::codes::*;
+use rustc_errors::{EmissionGuarantee, MultiSpan};
 use rustc_hir::def::{CtorKind, DefKind};
 use rustc_hir::{LangItem, Node, intravisit};
 use rustc_infer::infer::{RegionVariableOrigin, TyCtxtInferExt};
 use rustc_infer::traits::{Obligation, ObligationCauseCode};
 use rustc_lint_defs::builtin::{
-    REPR_TRANSPARENT_EXTERNAL_PRIVATE_FIELDS, UNSUPPORTED_FN_PTR_CALLING_CONVENTIONS,
+    REPR_TRANSPARENT_EXTERNAL_PRIVATE_FIELDS, UNSUPPORTED_CALLING_CONVENTIONS,
+    UNSUPPORTED_FN_PTR_CALLING_CONVENTIONS,
 };
 use rustc_middle::hir::nested_filter;
 use rustc_middle::middle::resolve_bound_vars::ResolvedArg;
 use rustc_middle::middle::stability::EvalResult;
 use rustc_middle::ty::error::TypeErrorToStringExt;
 use rustc_middle::ty::layout::{LayoutError, MAX_SIMD_LANES};
-use rustc_middle::ty::util::{Discr, IntTypeExt};
+use rustc_middle::ty::util::Discr;
 use rustc_middle::ty::{
     AdtDef, BottomUpFolder, GenericArgKind, RegionKind, TypeFoldable, TypeSuperVisitable,
     TypeVisitable, TypeVisitableExt, fold_regions,
 };
 use rustc_session::lint::builtin::UNINHABITED_STATIC;
+use rustc_target::spec::{AbiMap, AbiMapping};
 use rustc_trait_selection::error_reporting::InferCtxtErrorExt;
 use rustc_trait_selection::error_reporting::traits::on_unimplemented::OnUnimplementedDirective;
 use rustc_trait_selection::traits;
@@ -35,25 +37,56 @@ use {rustc_attr_data_structures as attrs, rustc_hir as hir};
 use super::compare_impl_item::check_type_bounds;
 use super::*;
 
-pub fn check_abi(tcx: TyCtxt<'_>, span: Span, abi: ExternAbi) {
-    if !tcx.sess.target.is_abi_supported(abi) {
-        struct_span_code_err!(
-            tcx.dcx(),
-            span,
-            E0570,
-            "`{abi}` is not a supported ABI for the current target",
-        )
-        .emit();
+pub fn check_abi(tcx: TyCtxt<'_>, hir_id: hir::HirId, span: Span, abi: ExternAbi) {
+    // FIXME: this should be checked earlier, e.g. in `rustc_ast_lowering`, to fix
+    // things like #86232.
+    fn add_help<T: EmissionGuarantee>(abi: ExternAbi, diag: &mut Diag<'_, T>) {
+        if let ExternAbi::Cdecl { unwind } = abi {
+            let c_abi = ExternAbi::C { unwind };
+            diag.help(format!("use `extern {c_abi}` instead",));
+        } else if let ExternAbi::Stdcall { unwind } = abi {
+            let c_abi = ExternAbi::C { unwind };
+            let system_abi = ExternAbi::System { unwind };
+            diag.help(format!(
+                "if you need `extern {abi}` on win32 and `extern {c_abi}` everywhere else, \
+                use `extern {system_abi}`"
+            ));
+        }
+    }
+
+    match AbiMap::from_target(&tcx.sess.target).canonize_abi(abi, false) {
+        AbiMapping::Direct(..) => (),
+        AbiMapping::Invalid => {
+            let mut err = struct_span_code_err!(
+                tcx.dcx(),
+                span,
+                E0570,
+                "`{abi}` is not a supported ABI for the current target",
+            );
+            add_help(abi, &mut err);
+            err.emit();
+        }
+        AbiMapping::Deprecated(..) => {
+            tcx.node_span_lint(UNSUPPORTED_CALLING_CONVENTIONS, hir_id, span, |lint| {
+                lint.primary_message("use of calling convention not supported on this target");
+                add_help(abi, lint);
+            });
+        }
     }
 }
 
 pub fn check_abi_fn_ptr(tcx: TyCtxt<'_>, hir_id: hir::HirId, span: Span, abi: ExternAbi) {
-    if !tcx.sess.target.is_abi_supported(abi) {
-        tcx.node_span_lint(UNSUPPORTED_FN_PTR_CALLING_CONVENTIONS, hir_id, span, |lint| {
-            lint.primary_message(format!(
-                "the calling convention {abi} is not supported on this target"
-            ));
-        });
+    // This is always an FCW, even for `AbiMapping::Invalid`, since we started linting later than
+    // in `check_abi` above.
+    match AbiMap::from_target(&tcx.sess.target).canonize_abi(abi, false) {
+        AbiMapping::Direct(..) => (),
+        AbiMapping::Deprecated(..) | AbiMapping::Invalid => {
+            tcx.node_span_lint(UNSUPPORTED_FN_PTR_CALLING_CONVENTIONS, hir_id, span, |lint| {
+                lint.primary_message(format!(
+                    "the calling convention {abi} is not supported on this target"
+                ));
+            });
+        }
     }
 }
 
@@ -97,7 +130,7 @@ fn allowed_union_or_unsafe_field<'tcx>(
     let def_id = tcx
         .lang_items()
         .get(LangItem::BikeshedGuaranteedNoDrop)
-        .unwrap_or_else(|| tcx.require_lang_item(LangItem::Copy, Some(span)));
+        .unwrap_or_else(|| tcx.require_lang_item(LangItem::Copy, span));
     let Ok(ty) = tcx.try_normalize_erasing_regions(typing_env, ty) else {
         tcx.dcx().span_delayed_bug(span, "could not normalize field type");
         return true;
@@ -702,6 +735,29 @@ fn check_static_linkage(tcx: TyCtxt<'_>, def_id: LocalDefId) {
 }
 
 pub(crate) fn check_item_type(tcx: TyCtxt<'_>, def_id: LocalDefId) {
+    let generics = tcx.generics_of(def_id);
+
+    for param in &generics.own_params {
+        match param.kind {
+            ty::GenericParamDefKind::Lifetime { .. } => {}
+            ty::GenericParamDefKind::Type { has_default, .. } => {
+                if has_default {
+                    tcx.ensure_ok().type_of(param.def_id);
+                }
+            }
+            ty::GenericParamDefKind::Const { has_default, .. } => {
+                tcx.ensure_ok().type_of(param.def_id);
+                if has_default {
+                    // need to store default and type of default
+                    let ct = tcx.const_param_default(param.def_id).skip_binder();
+                    if let ty::ConstKind::Unevaluated(uv) = ct.kind() {
+                        tcx.ensure_ok().type_of(uv.def);
+                    }
+                }
+            }
+        }
+    }
+
     match tcx.def_kind(def_id) {
         DefKind::Static { .. } => {
             check_static_inhabited(tcx, def_id);
@@ -770,6 +826,16 @@ pub(crate) fn check_item_type(tcx: TyCtxt<'_>, def_id: LocalDefId) {
             } else {
                 check_opaque(tcx, def_id);
             }
+
+            tcx.ensure_ok().predicates_of(def_id);
+            tcx.ensure_ok().explicit_item_bounds(def_id);
+            tcx.ensure_ok().explicit_item_self_bounds(def_id);
+            tcx.ensure_ok().item_bounds(def_id);
+            tcx.ensure_ok().item_self_bounds(def_id);
+            if tcx.is_conditionally_const(def_id) {
+                tcx.ensure_ok().explicit_implied_const_bounds(def_id);
+                tcx.ensure_ok().const_conditions(def_id);
+            }
         }
         DefKind::TyAlias => {
             check_type_alias_type_params_are_used(tcx, def_id);
@@ -779,7 +845,7 @@ pub(crate) fn check_item_type(tcx: TyCtxt<'_>, def_id: LocalDefId) {
             let hir::ItemKind::ForeignMod { abi, items } = it.kind else {
                 return;
             };
-            check_abi(tcx, it.span, abi);
+            check_abi(tcx, it.hir_id(), it.span, abi);
 
             for item in items {
                 let def_id = item.id.owner_id.def_id;
@@ -827,6 +893,15 @@ pub(crate) fn check_item_type(tcx: TyCtxt<'_>, def_id: LocalDefId) {
                 }
             }
         }
+        DefKind::Closure => {
+            // This is guaranteed to be called by metadata encoding,
+            // we still call it in wfcheck eagerly to ensure errors in codegen
+            // attrs prevent lints from spamming the output.
+            tcx.ensure_ok().codegen_fn_attrs(def_id);
+            // We do not call `type_of` for closures here as that
+            // depends on typecheck and would therefore hide
+            // any further errors in case one typeck fails.
+        }
         _ => {}
     }
 }
@@ -1100,7 +1175,7 @@ fn check_simd(tcx: TyCtxt<'_>, sp: Span, def_id: LocalDefId) {
             return;
         };
 
-        if let Some(second_field) = fields.get(FieldIdx::from_u32(1)) {
+        if let Some(second_field) = fields.get(FieldIdx::ONE) {
             struct_span_code_err!(tcx.dcx(), sp, E0075, "SIMD vector cannot have multiple fields")
                 .with_span_label(tcx.def_span(second_field.did), "excess field")
                 .emit();
@@ -1385,19 +1460,6 @@ fn check_enum(tcx: TyCtxt<'_>, def_id: LocalDefId) {
         );
     }
 
-    let repr_type_ty = def.repr().discr_type().to_ty(tcx);
-    if repr_type_ty == tcx.types.i128 || repr_type_ty == tcx.types.u128 {
-        if !tcx.features().repr128() {
-            feature_err(
-                &tcx.sess,
-                sym::repr128,
-                tcx.def_span(def_id),
-                "repr with 128-bit type is unstable",
-            )
-            .emit();
-        }
-    }
-
     for v in def.variants() {
         if let ty::VariantDiscr::Explicit(discr_def_id) = v.discr {
             tcx.ensure_ok().typeck(discr_def_id.expect_local());
@@ -1575,7 +1637,7 @@ fn check_type_alias_type_params_are_used<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalD
 
     let mut params_used = DenseBitSet::new_empty(generics.own_params.len());
     for leaf in ty.walk() {
-        if let GenericArgKind::Type(leaf_ty) = leaf.unpack()
+        if let GenericArgKind::Type(leaf_ty) = leaf.kind()
             && let ty::Param(param) = leaf_ty.kind()
         {
             debug!("found use of ty param {:?}", param);
@@ -1700,7 +1762,7 @@ fn opaque_type_cycle_error(tcx: TyCtxt<'_>, opaque_def_id: LocalDefId) -> ErrorG
 
                     let mut label_match = |ty: Ty<'_>, span| {
                         for arg in ty.walk() {
-                            if let ty::GenericArgKind::Type(ty) = arg.unpack()
+                            if let ty::GenericArgKind::Type(ty) = arg.kind()
                                 && let ty::Alias(
                                     ty::Opaque,
                                     ty::AliasTy { def_id: captured_def_id, .. },
diff --git a/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs b/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs
index 2ee41e2efbd..47681a78ecc 100644
--- a/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs
+++ b/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs
@@ -1231,7 +1231,7 @@ fn check_region_late_boundedness<'tcx>(
     for (id_arg, arg) in
         std::iter::zip(ty::GenericArgs::identity_for_item(tcx, impl_m.def_id), impl_m_args)
     {
-        if let ty::GenericArgKind::Lifetime(r) = arg.unpack()
+        if let ty::GenericArgKind::Lifetime(r) = arg.kind()
             && let ty::ReVar(vid) = r.kind()
             && let r = infcx
                 .inner
@@ -1256,7 +1256,7 @@ fn check_region_late_boundedness<'tcx>(
     for (id_arg, arg) in
         std::iter::zip(ty::GenericArgs::identity_for_item(tcx, trait_m.def_id), trait_m_args)
     {
-        if let ty::GenericArgKind::Lifetime(r) = arg.unpack()
+        if let ty::GenericArgKind::Lifetime(r) = arg.kind()
             && let ty::ReVar(vid) = r.kind()
             && let r = infcx
                 .inner
diff --git a/compiler/rustc_hir_analysis/src/check/compare_impl_item/refine.rs b/compiler/rustc_hir_analysis/src/check/compare_impl_item/refine.rs
index 4973d848959..3db1c40228f 100644
--- a/compiler/rustc_hir_analysis/src/check/compare_impl_item/refine.rs
+++ b/compiler/rustc_hir_analysis/src/check/compare_impl_item/refine.rs
@@ -427,7 +427,7 @@ fn report_mismatched_rpitit_captures<'tcx>(
     };
 
     trait_captured_args
-        .sort_by_cached_key(|arg| !matches!(arg.unpack(), ty::GenericArgKind::Lifetime(_)));
+        .sort_by_cached_key(|arg| !matches!(arg.kind(), ty::GenericArgKind::Lifetime(_)));
     let suggestion = format!("use<{}>", trait_captured_args.iter().join(", "));
 
     tcx.emit_node_span_lint(
diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs
index 9fd158ad154..481cdaa4c6c 100644
--- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs
+++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs
@@ -9,10 +9,7 @@ use rustc_span::def_id::LocalDefId;
 use rustc_span::{Span, Symbol, sym};
 
 use crate::check::check_function_signature;
-use crate::errors::{
-    UnrecognizedAtomicOperation, UnrecognizedIntrinsicFunction,
-    WrongNumberOfGenericArgumentsToIntrinsic,
-};
+use crate::errors::{UnrecognizedIntrinsicFunction, WrongNumberOfGenericArgumentsToIntrinsic};
 
 fn equate_intrinsic_type<'tcx>(
     tcx: TyCtxt<'tcx>,
@@ -172,7 +169,6 @@ pub(crate) fn check_intrinsic_type(
             Ty::new_error_with_message(tcx, span, "expected param")
         }
     };
-    let name_str = intrinsic_name.as_str();
 
     let bound_vars = tcx.mk_bound_variable_kinds(&[
         ty::BoundVariableKind::Region(ty::BoundRegionKind::Anon),
@@ -180,7 +176,7 @@ pub(crate) fn check_intrinsic_type(
         ty::BoundVariableKind::Region(ty::BoundRegionKind::ClosureEnv),
     ]);
     let mk_va_list_ty = |mutbl| {
-        let did = tcx.require_lang_item(LangItem::VaList, Some(span));
+        let did = tcx.require_lang_item(LangItem::VaList, span);
         let region = ty::Region::new_bound(
             tcx,
             ty::INNERMOST,
@@ -198,508 +194,471 @@ pub(crate) fn check_intrinsic_type(
         (Ty::new_ref(tcx, env_region, va_list_ty, mutbl), va_list_ty)
     };
 
-    let (n_tps, n_lts, n_cts, inputs, output, safety) = if name_str.starts_with("atomic_") {
-        let split: Vec<&str> = name_str.split('_').collect();
-        assert!(split.len() >= 2, "Atomic intrinsic in an incorrect format");
+    let safety = intrinsic_operation_unsafety(tcx, intrinsic_id);
+    let n_lts = 0;
+    let (n_tps, n_cts, inputs, output) = match intrinsic_name {
+        sym::abort => (0, 0, vec![], tcx.types.never),
+        sym::unreachable => (0, 0, vec![], tcx.types.never),
+        sym::breakpoint => (0, 0, vec![], tcx.types.unit),
+        sym::size_of | sym::pref_align_of | sym::min_align_of | sym::variant_count => {
+            (1, 0, vec![], tcx.types.usize)
+        }
+        sym::size_of_val | sym::min_align_of_val => {
+            (1, 0, vec![Ty::new_imm_ptr(tcx, param(0))], tcx.types.usize)
+        }
+        sym::rustc_peek => (1, 0, vec![param(0)], param(0)),
+        sym::caller_location => (0, 0, vec![], tcx.caller_location_ty()),
+        sym::assert_inhabited | sym::assert_zero_valid | sym::assert_mem_uninitialized_valid => {
+            (1, 0, vec![], tcx.types.unit)
+        }
+        sym::forget => (1, 0, vec![param(0)], tcx.types.unit),
+        sym::transmute | sym::transmute_unchecked => (2, 0, vec![param(0)], param(1)),
+        sym::prefetch_read_data
+        | sym::prefetch_write_data
+        | sym::prefetch_read_instruction
+        | sym::prefetch_write_instruction => {
+            (1, 0, vec![Ty::new_imm_ptr(tcx, param(0)), tcx.types.i32], tcx.types.unit)
+        }
+        sym::needs_drop => (1, 0, vec![], tcx.types.bool),
+
+        sym::type_name => (1, 0, vec![], Ty::new_static_str(tcx)),
+        sym::type_id => (1, 0, vec![], tcx.types.u128),
+        sym::offset => (2, 0, vec![param(0), param(1)], param(0)),
+        sym::arith_offset => (
+            1,
+            0,
+            vec![Ty::new_imm_ptr(tcx, param(0)), tcx.types.isize],
+            Ty::new_imm_ptr(tcx, param(0)),
+        ),
+        sym::slice_get_unchecked => (3, 0, vec![param(1), tcx.types.usize], param(0)),
+        sym::ptr_mask => (
+            1,
+            0,
+            vec![Ty::new_imm_ptr(tcx, param(0)), tcx.types.usize],
+            Ty::new_imm_ptr(tcx, param(0)),
+        ),
+
+        sym::copy | sym::copy_nonoverlapping => (
+            1,
+            0,
+            vec![Ty::new_imm_ptr(tcx, param(0)), Ty::new_mut_ptr(tcx, param(0)), tcx.types.usize],
+            tcx.types.unit,
+        ),
+        sym::volatile_copy_memory | sym::volatile_copy_nonoverlapping_memory => (
+            1,
+            0,
+            vec![Ty::new_mut_ptr(tcx, param(0)), Ty::new_imm_ptr(tcx, param(0)), tcx.types.usize],
+            tcx.types.unit,
+        ),
+        sym::compare_bytes => {
+            let byte_ptr = Ty::new_imm_ptr(tcx, tcx.types.u8);
+            (0, 0, vec![byte_ptr, byte_ptr, tcx.types.usize], tcx.types.i32)
+        }
+        sym::write_bytes | sym::volatile_set_memory => (
+            1,
+            0,
+            vec![Ty::new_mut_ptr(tcx, param(0)), tcx.types.u8, tcx.types.usize],
+            tcx.types.unit,
+        ),
+
+        sym::sqrtf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
+        sym::sqrtf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+        sym::sqrtf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+        sym::sqrtf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
+
+        sym::powif16 => (0, 0, vec![tcx.types.f16, tcx.types.i32], tcx.types.f16),
+        sym::powif32 => (0, 0, vec![tcx.types.f32, tcx.types.i32], tcx.types.f32),
+        sym::powif64 => (0, 0, vec![tcx.types.f64, tcx.types.i32], tcx.types.f64),
+        sym::powif128 => (0, 0, vec![tcx.types.f128, tcx.types.i32], tcx.types.f128),
+
+        sym::sinf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
+        sym::sinf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+        sym::sinf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+        sym::sinf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
+
+        sym::cosf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
+        sym::cosf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+        sym::cosf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+        sym::cosf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
+
+        sym::powf16 => (0, 0, vec![tcx.types.f16, tcx.types.f16], tcx.types.f16),
+        sym::powf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32),
+        sym::powf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64),
+        sym::powf128 => (0, 0, vec![tcx.types.f128, tcx.types.f128], tcx.types.f128),
+
+        sym::expf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
+        sym::expf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+        sym::expf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+        sym::expf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
+
+        sym::exp2f16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
+        sym::exp2f32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+        sym::exp2f64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+        sym::exp2f128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
+
+        sym::logf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
+        sym::logf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+        sym::logf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+        sym::logf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
+
+        sym::log10f16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
+        sym::log10f32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+        sym::log10f64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+        sym::log10f128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
+
+        sym::log2f16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
+        sym::log2f32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+        sym::log2f64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+        sym::log2f128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
+
+        sym::fmaf16 => (0, 0, vec![tcx.types.f16, tcx.types.f16, tcx.types.f16], tcx.types.f16),
+        sym::fmaf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32, tcx.types.f32], tcx.types.f32),
+        sym::fmaf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64, tcx.types.f64], tcx.types.f64),
+        sym::fmaf128 => {
+            (0, 0, vec![tcx.types.f128, tcx.types.f128, tcx.types.f128], tcx.types.f128)
+        }
 
-        // Each atomic op has variants with different suffixes (`_seq_cst`, `_acquire`, etc.). Use
-        // string ops to strip the suffixes, because the variants all get the same treatment here.
-        let (n_tps, inputs, output) = match split[1] {
-            "cxchg" | "cxchgweak" => (
-                1,
-                vec![Ty::new_mut_ptr(tcx, param(0)), param(0), param(0)],
-                Ty::new_tup(tcx, &[param(0), tcx.types.bool]),
-            ),
-            "load" => (1, vec![Ty::new_imm_ptr(tcx, param(0))], param(0)),
-            "store" => (1, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], tcx.types.unit),
-
-            "xchg" | "xadd" | "xsub" | "and" | "nand" | "or" | "xor" | "max" | "min" | "umax"
-            | "umin" => (1, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], param(0)),
-            "fence" | "singlethreadfence" => (0, Vec::new(), tcx.types.unit),
-            op => {
-                tcx.dcx().emit_err(UnrecognizedAtomicOperation { span, op });
-                return;
-            }
-        };
-        (n_tps, 0, 0, inputs, output, hir::Safety::Unsafe)
-    } else if intrinsic_name == sym::contract_check_ensures {
-        // contract_check_ensures::<Ret, C>(Ret, C) -> Ret
-        // where C: for<'a> Fn(&'a Ret) -> bool,
-        //
-        // so: two type params, 0 lifetime param, 0 const params, two inputs, no return
-        (2, 0, 0, vec![param(0), param(1)], param(1), hir::Safety::Safe)
-    } else {
-        let safety = intrinsic_operation_unsafety(tcx, intrinsic_id);
-        let (n_tps, n_cts, inputs, output) = match intrinsic_name {
-            sym::abort => (0, 0, vec![], tcx.types.never),
-            sym::unreachable => (0, 0, vec![], tcx.types.never),
-            sym::breakpoint => (0, 0, vec![], tcx.types.unit),
-            sym::size_of | sym::pref_align_of | sym::min_align_of | sym::variant_count => {
-                (1, 0, vec![], tcx.types.usize)
-            }
-            sym::size_of_val | sym::min_align_of_val => {
-                (1, 0, vec![Ty::new_imm_ptr(tcx, param(0))], tcx.types.usize)
-            }
-            sym::rustc_peek => (1, 0, vec![param(0)], param(0)),
-            sym::caller_location => (0, 0, vec![], tcx.caller_location_ty()),
-            sym::assert_inhabited
-            | sym::assert_zero_valid
-            | sym::assert_mem_uninitialized_valid => (1, 0, vec![], tcx.types.unit),
-            sym::forget => (1, 0, vec![param(0)], tcx.types.unit),
-            sym::transmute | sym::transmute_unchecked => (2, 0, vec![param(0)], param(1)),
-            sym::prefetch_read_data
-            | sym::prefetch_write_data
-            | sym::prefetch_read_instruction
-            | sym::prefetch_write_instruction => {
-                (1, 0, vec![Ty::new_imm_ptr(tcx, param(0)), tcx.types.i32], tcx.types.unit)
-            }
-            sym::needs_drop => (1, 0, vec![], tcx.types.bool),
-
-            sym::type_name => (1, 0, vec![], Ty::new_static_str(tcx)),
-            sym::type_id => (1, 0, vec![], tcx.types.u128),
-            sym::offset => (2, 0, vec![param(0), param(1)], param(0)),
-            sym::arith_offset => (
-                1,
-                0,
-                vec![Ty::new_imm_ptr(tcx, param(0)), tcx.types.isize],
-                Ty::new_imm_ptr(tcx, param(0)),
-            ),
-            sym::ptr_mask => (
-                1,
-                0,
-                vec![Ty::new_imm_ptr(tcx, param(0)), tcx.types.usize],
-                Ty::new_imm_ptr(tcx, param(0)),
-            ),
+        sym::fmuladdf16 => (0, 0, vec![tcx.types.f16, tcx.types.f16, tcx.types.f16], tcx.types.f16),
+        sym::fmuladdf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32, tcx.types.f32], tcx.types.f32),
+        sym::fmuladdf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64, tcx.types.f64], tcx.types.f64),
+        sym::fmuladdf128 => {
+            (0, 0, vec![tcx.types.f128, tcx.types.f128, tcx.types.f128], tcx.types.f128)
+        }
 
-            sym::copy | sym::copy_nonoverlapping => (
-                1,
-                0,
-                vec![
-                    Ty::new_imm_ptr(tcx, param(0)),
-                    Ty::new_mut_ptr(tcx, param(0)),
-                    tcx.types.usize,
-                ],
-                tcx.types.unit,
-            ),
-            sym::volatile_copy_memory | sym::volatile_copy_nonoverlapping_memory => (
+        sym::fabsf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
+        sym::fabsf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+        sym::fabsf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+        sym::fabsf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
+
+        sym::minnumf16 => (0, 0, vec![tcx.types.f16, tcx.types.f16], tcx.types.f16),
+        sym::minnumf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32),
+        sym::minnumf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64),
+        sym::minnumf128 => (0, 0, vec![tcx.types.f128, tcx.types.f128], tcx.types.f128),
+
+        sym::minimumf16 => (0, 0, vec![tcx.types.f16, tcx.types.f16], tcx.types.f16),
+        sym::minimumf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32),
+        sym::minimumf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64),
+        sym::minimumf128 => (0, 0, vec![tcx.types.f128, tcx.types.f128], tcx.types.f128),
+
+        sym::maxnumf16 => (0, 0, vec![tcx.types.f16, tcx.types.f16], tcx.types.f16),
+        sym::maxnumf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32),
+        sym::maxnumf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64),
+        sym::maxnumf128 => (0, 0, vec![tcx.types.f128, tcx.types.f128], tcx.types.f128),
+
+        sym::maximumf16 => (0, 0, vec![tcx.types.f16, tcx.types.f16], tcx.types.f16),
+        sym::maximumf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32),
+        sym::maximumf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64),
+        sym::maximumf128 => (0, 0, vec![tcx.types.f128, tcx.types.f128], tcx.types.f128),
+
+        sym::copysignf16 => (0, 0, vec![tcx.types.f16, tcx.types.f16], tcx.types.f16),
+        sym::copysignf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32),
+        sym::copysignf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64),
+        sym::copysignf128 => (0, 0, vec![tcx.types.f128, tcx.types.f128], tcx.types.f128),
+
+        sym::floorf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
+        sym::floorf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+        sym::floorf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+        sym::floorf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
+
+        sym::ceilf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
+        sym::ceilf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+        sym::ceilf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+        sym::ceilf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
+
+        sym::truncf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
+        sym::truncf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+        sym::truncf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+        sym::truncf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
+
+        sym::round_ties_even_f16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
+        sym::round_ties_even_f32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+        sym::round_ties_even_f64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+        sym::round_ties_even_f128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
+
+        sym::roundf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
+        sym::roundf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
+        sym::roundf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
+        sym::roundf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
+
+        sym::volatile_load | sym::unaligned_volatile_load => {
+            (1, 0, vec![Ty::new_imm_ptr(tcx, param(0))], param(0))
+        }
+        sym::volatile_store | sym::unaligned_volatile_store => {
+            (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], tcx.types.unit)
+        }
+
+        sym::ctpop | sym::ctlz | sym::ctlz_nonzero | sym::cttz | sym::cttz_nonzero => {
+            (1, 0, vec![param(0)], tcx.types.u32)
+        }
+
+        sym::bswap | sym::bitreverse => (1, 0, vec![param(0)], param(0)),
+
+        sym::three_way_compare => (1, 0, vec![param(0), param(0)], tcx.ty_ordering_enum(span)),
+
+        sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
+            (1, 0, vec![param(0), param(0)], Ty::new_tup(tcx, &[param(0), tcx.types.bool]))
+        }
+
+        sym::carrying_mul_add => (2, 0, vec![param(0); 4], Ty::new_tup(tcx, &[param(1), param(0)])),
+
+        sym::ptr_guaranteed_cmp => (
+            1,
+            0,
+            vec![Ty::new_imm_ptr(tcx, param(0)), Ty::new_imm_ptr(tcx, param(0))],
+            tcx.types.u8,
+        ),
+
+        sym::const_allocate => {
+            (0, 0, vec![tcx.types.usize, tcx.types.usize], Ty::new_mut_ptr(tcx, tcx.types.u8))
+        }
+        sym::const_deallocate => (
+            0,
+            0,
+            vec![Ty::new_mut_ptr(tcx, tcx.types.u8), tcx.types.usize, tcx.types.usize],
+            tcx.types.unit,
+        ),
+
+        sym::ptr_offset_from => (
+            1,
+            0,
+            vec![Ty::new_imm_ptr(tcx, param(0)), Ty::new_imm_ptr(tcx, param(0))],
+            tcx.types.isize,
+        ),
+        sym::ptr_offset_from_unsigned => (
+            1,
+            0,
+            vec![Ty::new_imm_ptr(tcx, param(0)), Ty::new_imm_ptr(tcx, param(0))],
+            tcx.types.usize,
+        ),
+        sym::unchecked_div | sym::unchecked_rem | sym::exact_div | sym::disjoint_bitor => {
+            (1, 0, vec![param(0), param(0)], param(0))
+        }
+        sym::unchecked_shl | sym::unchecked_shr => (2, 0, vec![param(0), param(1)], param(0)),
+        sym::rotate_left | sym::rotate_right => (1, 0, vec![param(0), tcx.types.u32], param(0)),
+        sym::unchecked_add | sym::unchecked_sub | sym::unchecked_mul => {
+            (1, 0, vec![param(0), param(0)], param(0))
+        }
+        sym::wrapping_add | sym::wrapping_sub | sym::wrapping_mul => {
+            (1, 0, vec![param(0), param(0)], param(0))
+        }
+        sym::saturating_add | sym::saturating_sub => (1, 0, vec![param(0), param(0)], param(0)),
+        sym::fadd_fast | sym::fsub_fast | sym::fmul_fast | sym::fdiv_fast | sym::frem_fast => {
+            (1, 0, vec![param(0), param(0)], param(0))
+        }
+        sym::fadd_algebraic
+        | sym::fsub_algebraic
+        | sym::fmul_algebraic
+        | sym::fdiv_algebraic
+        | sym::frem_algebraic => (1, 0, vec![param(0), param(0)], param(0)),
+        sym::float_to_int_unchecked => (2, 0, vec![param(0)], param(1)),
+
+        sym::assume => (0, 0, vec![tcx.types.bool], tcx.types.unit),
+        sym::select_unpredictable => (1, 0, vec![tcx.types.bool, param(0), param(0)], param(0)),
+        sym::cold_path => (0, 0, vec![], tcx.types.unit),
+
+        sym::read_via_copy => (1, 0, vec![Ty::new_imm_ptr(tcx, param(0))], param(0)),
+        sym::write_via_move => {
+            (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], tcx.types.unit)
+        }
+
+        sym::typed_swap_nonoverlapping => {
+            (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)); 2], tcx.types.unit)
+        }
+
+        sym::discriminant_value => {
+            let assoc_items = tcx.associated_item_def_ids(
+                tcx.require_lang_item(hir::LangItem::DiscriminantKind, span),
+            );
+            let discriminant_def_id = assoc_items[0];
+
+            let br = ty::BoundRegion { var: ty::BoundVar::ZERO, kind: ty::BoundRegionKind::Anon };
+            (
                 1,
                 0,
-                vec![
-                    Ty::new_mut_ptr(tcx, param(0)),
-                    Ty::new_imm_ptr(tcx, param(0)),
-                    tcx.types.usize,
-                ],
+                vec![Ty::new_imm_ref(tcx, ty::Region::new_bound(tcx, ty::INNERMOST, br), param(0))],
+                Ty::new_projection_from_args(
+                    tcx,
+                    discriminant_def_id,
+                    tcx.mk_args(&[param(0).into()]),
+                ),
+            )
+        }
+
+        sym::catch_unwind => {
+            let mut_u8 = Ty::new_mut_ptr(tcx, tcx.types.u8);
+            let try_fn_ty = ty::Binder::dummy(tcx.mk_fn_sig(
+                [mut_u8],
                 tcx.types.unit,
-            ),
-            sym::compare_bytes => {
-                let byte_ptr = Ty::new_imm_ptr(tcx, tcx.types.u8);
-                (0, 0, vec![byte_ptr, byte_ptr, tcx.types.usize], tcx.types.i32)
-            }
-            sym::write_bytes | sym::volatile_set_memory => (
-                1,
-                0,
-                vec![Ty::new_mut_ptr(tcx, param(0)), tcx.types.u8, tcx.types.usize],
+                false,
+                hir::Safety::Safe,
+                ExternAbi::Rust,
+            ));
+            let catch_fn_ty = ty::Binder::dummy(tcx.mk_fn_sig(
+                [mut_u8, mut_u8],
                 tcx.types.unit,
-            ),
-
-            sym::sqrtf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
-            sym::sqrtf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
-            sym::sqrtf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
-            sym::sqrtf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
-
-            sym::powif16 => (0, 0, vec![tcx.types.f16, tcx.types.i32], tcx.types.f16),
-            sym::powif32 => (0, 0, vec![tcx.types.f32, tcx.types.i32], tcx.types.f32),
-            sym::powif64 => (0, 0, vec![tcx.types.f64, tcx.types.i32], tcx.types.f64),
-            sym::powif128 => (0, 0, vec![tcx.types.f128, tcx.types.i32], tcx.types.f128),
-
-            sym::sinf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
-            sym::sinf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
-            sym::sinf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
-            sym::sinf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
-
-            sym::cosf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
-            sym::cosf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
-            sym::cosf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
-            sym::cosf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
-
-            sym::powf16 => (0, 0, vec![tcx.types.f16, tcx.types.f16], tcx.types.f16),
-            sym::powf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32),
-            sym::powf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64),
-            sym::powf128 => (0, 0, vec![tcx.types.f128, tcx.types.f128], tcx.types.f128),
-
-            sym::expf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
-            sym::expf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
-            sym::expf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
-            sym::expf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
-
-            sym::exp2f16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
-            sym::exp2f32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
-            sym::exp2f64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
-            sym::exp2f128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
-
-            sym::logf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
-            sym::logf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
-            sym::logf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
-            sym::logf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
-
-            sym::log10f16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
-            sym::log10f32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
-            sym::log10f64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
-            sym::log10f128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
-
-            sym::log2f16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
-            sym::log2f32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
-            sym::log2f64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
-            sym::log2f128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
-
-            sym::fmaf16 => (0, 0, vec![tcx.types.f16, tcx.types.f16, tcx.types.f16], tcx.types.f16),
-            sym::fmaf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32, tcx.types.f32], tcx.types.f32),
-            sym::fmaf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64, tcx.types.f64], tcx.types.f64),
-            sym::fmaf128 => {
-                (0, 0, vec![tcx.types.f128, tcx.types.f128, tcx.types.f128], tcx.types.f128)
-            }
-
-            sym::fmuladdf16 => {
-                (0, 0, vec![tcx.types.f16, tcx.types.f16, tcx.types.f16], tcx.types.f16)
-            }
-            sym::fmuladdf32 => {
-                (0, 0, vec![tcx.types.f32, tcx.types.f32, tcx.types.f32], tcx.types.f32)
-            }
-            sym::fmuladdf64 => {
-                (0, 0, vec![tcx.types.f64, tcx.types.f64, tcx.types.f64], tcx.types.f64)
-            }
-            sym::fmuladdf128 => {
-                (0, 0, vec![tcx.types.f128, tcx.types.f128, tcx.types.f128], tcx.types.f128)
-            }
-
-            sym::fabsf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
-            sym::fabsf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
-            sym::fabsf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
-            sym::fabsf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
-
-            sym::minnumf16 => (0, 0, vec![tcx.types.f16, tcx.types.f16], tcx.types.f16),
-            sym::minnumf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32),
-            sym::minnumf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64),
-            sym::minnumf128 => (0, 0, vec![tcx.types.f128, tcx.types.f128], tcx.types.f128),
-
-            sym::minimumf16 => (0, 0, vec![tcx.types.f16, tcx.types.f16], tcx.types.f16),
-            sym::minimumf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32),
-            sym::minimumf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64),
-            sym::minimumf128 => (0, 0, vec![tcx.types.f128, tcx.types.f128], tcx.types.f128),
-
-            sym::maxnumf16 => (0, 0, vec![tcx.types.f16, tcx.types.f16], tcx.types.f16),
-            sym::maxnumf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32),
-            sym::maxnumf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64),
-            sym::maxnumf128 => (0, 0, vec![tcx.types.f128, tcx.types.f128], tcx.types.f128),
-
-            sym::maximumf16 => (0, 0, vec![tcx.types.f16, tcx.types.f16], tcx.types.f16),
-            sym::maximumf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32),
-            sym::maximumf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64),
-            sym::maximumf128 => (0, 0, vec![tcx.types.f128, tcx.types.f128], tcx.types.f128),
-
-            sym::copysignf16 => (0, 0, vec![tcx.types.f16, tcx.types.f16], tcx.types.f16),
-            sym::copysignf32 => (0, 0, vec![tcx.types.f32, tcx.types.f32], tcx.types.f32),
-            sym::copysignf64 => (0, 0, vec![tcx.types.f64, tcx.types.f64], tcx.types.f64),
-            sym::copysignf128 => (0, 0, vec![tcx.types.f128, tcx.types.f128], tcx.types.f128),
-
-            sym::floorf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
-            sym::floorf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
-            sym::floorf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
-            sym::floorf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
-
-            sym::ceilf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
-            sym::ceilf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
-            sym::ceilf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
-            sym::ceilf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
-
-            sym::truncf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
-            sym::truncf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
-            sym::truncf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
-            sym::truncf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
-
-            sym::round_ties_even_f16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
-            sym::round_ties_even_f32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
-            sym::round_ties_even_f64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
-            sym::round_ties_even_f128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
-
-            sym::roundf16 => (0, 0, vec![tcx.types.f16], tcx.types.f16),
-            sym::roundf32 => (0, 0, vec![tcx.types.f32], tcx.types.f32),
-            sym::roundf64 => (0, 0, vec![tcx.types.f64], tcx.types.f64),
-            sym::roundf128 => (0, 0, vec![tcx.types.f128], tcx.types.f128),
-
-            sym::volatile_load | sym::unaligned_volatile_load => {
-                (1, 0, vec![Ty::new_imm_ptr(tcx, param(0))], param(0))
-            }
-            sym::volatile_store | sym::unaligned_volatile_store => {
-                (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], tcx.types.unit)
-            }
-
-            sym::ctpop | sym::ctlz | sym::ctlz_nonzero | sym::cttz | sym::cttz_nonzero => {
-                (1, 0, vec![param(0)], tcx.types.u32)
-            }
-
-            sym::bswap | sym::bitreverse => (1, 0, vec![param(0)], param(0)),
-
-            sym::three_way_compare => {
-                (1, 0, vec![param(0), param(0)], tcx.ty_ordering_enum(Some(span)))
-            }
-
-            sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
-                (1, 0, vec![param(0), param(0)], Ty::new_tup(tcx, &[param(0), tcx.types.bool]))
-            }
-
-            sym::carrying_mul_add => {
-                (2, 0, vec![param(0); 4], Ty::new_tup(tcx, &[param(1), param(0)]))
-            }
-
-            sym::ptr_guaranteed_cmp => (
-                1,
-                0,
-                vec![Ty::new_imm_ptr(tcx, param(0)), Ty::new_imm_ptr(tcx, param(0))],
-                tcx.types.u8,
-            ),
-
-            sym::const_allocate => {
-                (0, 0, vec![tcx.types.usize, tcx.types.usize], Ty::new_mut_ptr(tcx, tcx.types.u8))
-            }
-            sym::const_deallocate => (
+                false,
+                hir::Safety::Safe,
+                ExternAbi::Rust,
+            ));
+            (
                 0,
                 0,
-                vec![Ty::new_mut_ptr(tcx, tcx.types.u8), tcx.types.usize, tcx.types.usize],
-                tcx.types.unit,
-            ),
+                vec![Ty::new_fn_ptr(tcx, try_fn_ty), mut_u8, Ty::new_fn_ptr(tcx, catch_fn_ty)],
+                tcx.types.i32,
+            )
+        }
 
-            sym::ptr_offset_from => (
-                1,
-                0,
-                vec![Ty::new_imm_ptr(tcx, param(0)), Ty::new_imm_ptr(tcx, param(0))],
-                tcx.types.isize,
-            ),
-            sym::ptr_offset_from_unsigned => (
-                1,
-                0,
-                vec![Ty::new_imm_ptr(tcx, param(0)), Ty::new_imm_ptr(tcx, param(0))],
-                tcx.types.usize,
-            ),
-            sym::unchecked_div | sym::unchecked_rem | sym::exact_div | sym::disjoint_bitor => {
-                (1, 0, vec![param(0), param(0)], param(0))
-            }
-            sym::unchecked_shl | sym::unchecked_shr => (2, 0, vec![param(0), param(1)], param(0)),
-            sym::rotate_left | sym::rotate_right => (1, 0, vec![param(0), tcx.types.u32], param(0)),
-            sym::unchecked_add | sym::unchecked_sub | sym::unchecked_mul => {
-                (1, 0, vec![param(0), param(0)], param(0))
-            }
-            sym::wrapping_add | sym::wrapping_sub | sym::wrapping_mul => {
-                (1, 0, vec![param(0), param(0)], param(0))
-            }
-            sym::saturating_add | sym::saturating_sub => (1, 0, vec![param(0), param(0)], param(0)),
-            sym::fadd_fast | sym::fsub_fast | sym::fmul_fast | sym::fdiv_fast | sym::frem_fast => {
-                (1, 0, vec![param(0), param(0)], param(0))
-            }
-            sym::fadd_algebraic
-            | sym::fsub_algebraic
-            | sym::fmul_algebraic
-            | sym::fdiv_algebraic
-            | sym::frem_algebraic => (1, 0, vec![param(0), param(0)], param(0)),
-            sym::float_to_int_unchecked => (2, 0, vec![param(0)], param(1)),
-
-            sym::assume => (0, 0, vec![tcx.types.bool], tcx.types.unit),
-            sym::select_unpredictable => (1, 0, vec![tcx.types.bool, param(0), param(0)], param(0)),
-            sym::cold_path => (0, 0, vec![], tcx.types.unit),
-
-            sym::read_via_copy => (1, 0, vec![Ty::new_imm_ptr(tcx, param(0))], param(0)),
-            sym::write_via_move => {
-                (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], tcx.types.unit)
-            }
-
-            sym::typed_swap_nonoverlapping => {
-                (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)); 2], tcx.types.unit)
-            }
-
-            sym::discriminant_value => {
-                let assoc_items = tcx.associated_item_def_ids(
-                    tcx.require_lang_item(hir::LangItem::DiscriminantKind, None),
-                );
-                let discriminant_def_id = assoc_items[0];
-
-                let br =
-                    ty::BoundRegion { var: ty::BoundVar::ZERO, kind: ty::BoundRegionKind::Anon };
-                (
-                    1,
-                    0,
-                    vec![Ty::new_imm_ref(
-                        tcx,
-                        ty::Region::new_bound(tcx, ty::INNERMOST, br),
-                        param(0),
-                    )],
-                    Ty::new_projection_from_args(
-                        tcx,
-                        discriminant_def_id,
-                        tcx.mk_args(&[param(0).into()]),
-                    ),
-                )
-            }
-
-            sym::catch_unwind => {
-                let mut_u8 = Ty::new_mut_ptr(tcx, tcx.types.u8);
-                let try_fn_ty = ty::Binder::dummy(tcx.mk_fn_sig(
-                    [mut_u8],
-                    tcx.types.unit,
-                    false,
-                    hir::Safety::Safe,
-                    ExternAbi::Rust,
-                ));
-                let catch_fn_ty = ty::Binder::dummy(tcx.mk_fn_sig(
-                    [mut_u8, mut_u8],
-                    tcx.types.unit,
-                    false,
-                    hir::Safety::Safe,
-                    ExternAbi::Rust,
-                ));
-                (
-                    0,
-                    0,
-                    vec![Ty::new_fn_ptr(tcx, try_fn_ty), mut_u8, Ty::new_fn_ptr(tcx, catch_fn_ty)],
-                    tcx.types.i32,
-                )
-            }
-
-            sym::va_start | sym::va_end => {
-                (0, 0, vec![mk_va_list_ty(hir::Mutability::Mut).0], tcx.types.unit)
-            }
-
-            sym::va_copy => {
-                let (va_list_ref_ty, va_list_ty) = mk_va_list_ty(hir::Mutability::Not);
-                let va_list_ptr_ty = Ty::new_mut_ptr(tcx, va_list_ty);
-                (0, 0, vec![va_list_ptr_ty, va_list_ref_ty], tcx.types.unit)
-            }
-
-            sym::va_arg => (1, 0, vec![mk_va_list_ty(hir::Mutability::Mut).0], param(0)),
-
-            sym::nontemporal_store => {
-                (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], tcx.types.unit)
-            }
-
-            sym::raw_eq => {
-                let br =
-                    ty::BoundRegion { var: ty::BoundVar::ZERO, kind: ty::BoundRegionKind::Anon };
-                let param_ty_lhs =
-                    Ty::new_imm_ref(tcx, ty::Region::new_bound(tcx, ty::INNERMOST, br), param(0));
-                let br = ty::BoundRegion {
-                    var: ty::BoundVar::from_u32(1),
-                    kind: ty::BoundRegionKind::Anon,
-                };
-                let param_ty_rhs =
-                    Ty::new_imm_ref(tcx, ty::Region::new_bound(tcx, ty::INNERMOST, br), param(0));
-                (1, 0, vec![param_ty_lhs, param_ty_rhs], tcx.types.bool)
-            }
-
-            sym::black_box => (1, 0, vec![param(0)], param(0)),
-
-            sym::is_val_statically_known => (1, 0, vec![param(0)], tcx.types.bool),
-
-            sym::const_eval_select => (4, 0, vec![param(0), param(1), param(2)], param(3)),
-
-            sym::vtable_size | sym::vtable_align => {
-                (0, 0, vec![Ty::new_imm_ptr(tcx, tcx.types.unit)], tcx.types.usize)
-            }
-
-            // This type check is not particularly useful, but the `where` bounds
-            // on the definition in `core` do the heavy lifting for checking it.
-            sym::aggregate_raw_ptr => (3, 0, vec![param(1), param(2)], param(0)),
-            sym::ptr_metadata => (2, 0, vec![Ty::new_imm_ptr(tcx, param(0))], param(1)),
-
-            sym::ub_checks => (0, 0, Vec::new(), tcx.types.bool),
-
-            sym::box_new => (1, 0, vec![param(0)], Ty::new_box(tcx, param(0))),
-
-            // contract_checks() -> bool
-            sym::contract_checks => (0, 0, Vec::new(), tcx.types.bool),
-            // contract_check_requires::<C>(C) -> bool, where C: impl Fn() -> bool
-            sym::contract_check_requires => (1, 0, vec![param(0)], tcx.types.unit),
-
-            sym::simd_eq
-            | sym::simd_ne
-            | sym::simd_lt
-            | sym::simd_le
-            | sym::simd_gt
-            | sym::simd_ge => (2, 0, vec![param(0), param(0)], param(1)),
-            sym::simd_add
-            | sym::simd_sub
-            | sym::simd_mul
-            | sym::simd_rem
-            | sym::simd_div
-            | sym::simd_shl
-            | sym::simd_shr
-            | sym::simd_and
-            | sym::simd_or
-            | sym::simd_xor
-            | sym::simd_fmin
-            | sym::simd_fmax
-            | sym::simd_saturating_add
-            | sym::simd_saturating_sub => (1, 0, vec![param(0), param(0)], param(0)),
-            sym::simd_arith_offset => (2, 0, vec![param(0), param(1)], param(0)),
-            sym::simd_neg
-            | sym::simd_bswap
-            | sym::simd_bitreverse
-            | sym::simd_ctlz
-            | sym::simd_cttz
-            | sym::simd_ctpop
-            | sym::simd_fsqrt
-            | sym::simd_fsin
-            | sym::simd_fcos
-            | sym::simd_fexp
-            | sym::simd_fexp2
-            | sym::simd_flog2
-            | sym::simd_flog10
-            | sym::simd_flog
-            | sym::simd_fabs
-            | sym::simd_ceil
-            | sym::simd_floor
-            | sym::simd_round
-            | sym::simd_trunc => (1, 0, vec![param(0)], param(0)),
-            sym::simd_fma | sym::simd_relaxed_fma => {
-                (1, 0, vec![param(0), param(0), param(0)], param(0))
-            }
-            sym::simd_gather => (3, 0, vec![param(0), param(1), param(2)], param(0)),
-            sym::simd_masked_load => (3, 0, vec![param(0), param(1), param(2)], param(2)),
-            sym::simd_masked_store => (3, 0, vec![param(0), param(1), param(2)], tcx.types.unit),
-            sym::simd_scatter => (3, 0, vec![param(0), param(1), param(2)], tcx.types.unit),
-            sym::simd_insert | sym::simd_insert_dyn => {
-                (2, 0, vec![param(0), tcx.types.u32, param(1)], param(0))
-            }
-            sym::simd_extract | sym::simd_extract_dyn => {
-                (2, 0, vec![param(0), tcx.types.u32], param(1))
-            }
-            sym::simd_cast
-            | sym::simd_as
-            | sym::simd_cast_ptr
-            | sym::simd_expose_provenance
-            | sym::simd_with_exposed_provenance => (2, 0, vec![param(0)], param(1)),
-            sym::simd_bitmask => (2, 0, vec![param(0)], param(1)),
-            sym::simd_select | sym::simd_select_bitmask => {
-                (2, 0, vec![param(0), param(1), param(1)], param(1))
-            }
-            sym::simd_reduce_all | sym::simd_reduce_any => (1, 0, vec![param(0)], tcx.types.bool),
-            sym::simd_reduce_add_ordered | sym::simd_reduce_mul_ordered => {
-                (2, 0, vec![param(0), param(1)], param(1))
-            }
-            sym::simd_reduce_add_unordered
-            | sym::simd_reduce_mul_unordered
-            | sym::simd_reduce_and
-            | sym::simd_reduce_or
-            | sym::simd_reduce_xor
-            | sym::simd_reduce_min
-            | sym::simd_reduce_max => (2, 0, vec![param(0)], param(1)),
-            sym::simd_shuffle => (3, 0, vec![param(0), param(0), param(1)], param(2)),
-            sym::simd_shuffle_const_generic => (2, 1, vec![param(0), param(0)], param(1)),
-
-            other => {
-                tcx.dcx().emit_err(UnrecognizedIntrinsicFunction { span, name: other });
-                return;
-            }
-        };
-        (n_tps, 0, n_cts, inputs, output, safety)
+        sym::va_start | sym::va_end => {
+            (0, 0, vec![mk_va_list_ty(hir::Mutability::Mut).0], tcx.types.unit)
+        }
+
+        sym::va_copy => {
+            let (va_list_ref_ty, va_list_ty) = mk_va_list_ty(hir::Mutability::Not);
+            let va_list_ptr_ty = Ty::new_mut_ptr(tcx, va_list_ty);
+            (0, 0, vec![va_list_ptr_ty, va_list_ref_ty], tcx.types.unit)
+        }
+
+        sym::va_arg => (1, 0, vec![mk_va_list_ty(hir::Mutability::Mut).0], param(0)),
+
+        sym::nontemporal_store => {
+            (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], tcx.types.unit)
+        }
+
+        sym::raw_eq => {
+            let br = ty::BoundRegion { var: ty::BoundVar::ZERO, kind: ty::BoundRegionKind::Anon };
+            let param_ty_lhs =
+                Ty::new_imm_ref(tcx, ty::Region::new_bound(tcx, ty::INNERMOST, br), param(0));
+            let br =
+                ty::BoundRegion { var: ty::BoundVar::from_u32(1), kind: ty::BoundRegionKind::Anon };
+            let param_ty_rhs =
+                Ty::new_imm_ref(tcx, ty::Region::new_bound(tcx, ty::INNERMOST, br), param(0));
+            (1, 0, vec![param_ty_lhs, param_ty_rhs], tcx.types.bool)
+        }
+
+        sym::black_box => (1, 0, vec![param(0)], param(0)),
+
+        sym::is_val_statically_known => (1, 0, vec![param(0)], tcx.types.bool),
+
+        sym::const_eval_select => (4, 0, vec![param(0), param(1), param(2)], param(3)),
+
+        sym::vtable_size | sym::vtable_align => {
+            (0, 0, vec![Ty::new_imm_ptr(tcx, tcx.types.unit)], tcx.types.usize)
+        }
+
+        // This type check is not particularly useful, but the `where` bounds
+        // on the definition in `core` do the heavy lifting for checking it.
+        sym::aggregate_raw_ptr => (3, 0, vec![param(1), param(2)], param(0)),
+        sym::ptr_metadata => (2, 0, vec![Ty::new_imm_ptr(tcx, param(0))], param(1)),
+
+        sym::ub_checks => (0, 0, Vec::new(), tcx.types.bool),
+
+        sym::box_new => (1, 0, vec![param(0)], Ty::new_box(tcx, param(0))),
+
+        // contract_checks() -> bool
+        sym::contract_checks => (0, 0, Vec::new(), tcx.types.bool),
+        // contract_check_requires::<C>(C) -> bool, where C: impl Fn() -> bool
+        sym::contract_check_requires => (1, 0, vec![param(0)], tcx.types.unit),
+        sym::contract_check_ensures => (2, 0, vec![param(0), param(1)], param(1)),
+
+        sym::simd_eq | sym::simd_ne | sym::simd_lt | sym::simd_le | sym::simd_gt | sym::simd_ge => {
+            (2, 0, vec![param(0), param(0)], param(1))
+        }
+        sym::simd_add
+        | sym::simd_sub
+        | sym::simd_mul
+        | sym::simd_rem
+        | sym::simd_div
+        | sym::simd_shl
+        | sym::simd_shr
+        | sym::simd_and
+        | sym::simd_or
+        | sym::simd_xor
+        | sym::simd_fmin
+        | sym::simd_fmax
+        | sym::simd_saturating_add
+        | sym::simd_saturating_sub => (1, 0, vec![param(0), param(0)], param(0)),
+        sym::simd_arith_offset => (2, 0, vec![param(0), param(1)], param(0)),
+        sym::simd_neg
+        | sym::simd_bswap
+        | sym::simd_bitreverse
+        | sym::simd_ctlz
+        | sym::simd_cttz
+        | sym::simd_ctpop
+        | sym::simd_fsqrt
+        | sym::simd_fsin
+        | sym::simd_fcos
+        | sym::simd_fexp
+        | sym::simd_fexp2
+        | sym::simd_flog2
+        | sym::simd_flog10
+        | sym::simd_flog
+        | sym::simd_fabs
+        | sym::simd_ceil
+        | sym::simd_floor
+        | sym::simd_round
+        | sym::simd_trunc => (1, 0, vec![param(0)], param(0)),
+        sym::simd_fma | sym::simd_relaxed_fma => {
+            (1, 0, vec![param(0), param(0), param(0)], param(0))
+        }
+        sym::simd_gather => (3, 0, vec![param(0), param(1), param(2)], param(0)),
+        sym::simd_masked_load => (3, 0, vec![param(0), param(1), param(2)], param(2)),
+        sym::simd_masked_store => (3, 0, vec![param(0), param(1), param(2)], tcx.types.unit),
+        sym::simd_scatter => (3, 0, vec![param(0), param(1), param(2)], tcx.types.unit),
+        sym::simd_insert | sym::simd_insert_dyn => {
+            (2, 0, vec![param(0), tcx.types.u32, param(1)], param(0))
+        }
+        sym::simd_extract | sym::simd_extract_dyn => {
+            (2, 0, vec![param(0), tcx.types.u32], param(1))
+        }
+        sym::simd_cast
+        | sym::simd_as
+        | sym::simd_cast_ptr
+        | sym::simd_expose_provenance
+        | sym::simd_with_exposed_provenance => (2, 0, vec![param(0)], param(1)),
+        sym::simd_bitmask => (2, 0, vec![param(0)], param(1)),
+        sym::simd_select | sym::simd_select_bitmask => {
+            (2, 0, vec![param(0), param(1), param(1)], param(1))
+        }
+        sym::simd_reduce_all | sym::simd_reduce_any => (1, 0, vec![param(0)], tcx.types.bool),
+        sym::simd_reduce_add_ordered | sym::simd_reduce_mul_ordered => {
+            (2, 0, vec![param(0), param(1)], param(1))
+        }
+        sym::simd_reduce_add_unordered
+        | sym::simd_reduce_mul_unordered
+        | sym::simd_reduce_and
+        | sym::simd_reduce_or
+        | sym::simd_reduce_xor
+        | sym::simd_reduce_min
+        | sym::simd_reduce_max => (2, 0, vec![param(0)], param(1)),
+        sym::simd_shuffle => (3, 0, vec![param(0), param(0), param(1)], param(2)),
+        sym::simd_shuffle_const_generic => (2, 1, vec![param(0), param(0)], param(1)),
+
+        sym::atomic_cxchg | sym::atomic_cxchgweak => (
+            1,
+            2,
+            vec![Ty::new_mut_ptr(tcx, param(0)), param(0), param(0)],
+            Ty::new_tup(tcx, &[param(0), tcx.types.bool]),
+        ),
+        sym::atomic_load => (1, 1, vec![Ty::new_imm_ptr(tcx, param(0))], param(0)),
+        sym::atomic_store => (1, 1, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], tcx.types.unit),
+
+        sym::atomic_xchg
+        | sym::atomic_xadd
+        | sym::atomic_xsub
+        | sym::atomic_and
+        | sym::atomic_nand
+        | sym::atomic_or
+        | sym::atomic_xor
+        | sym::atomic_max
+        | sym::atomic_min
+        | sym::atomic_umax
+        | sym::atomic_umin => (1, 1, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], param(0)),
+        sym::atomic_fence | sym::atomic_singlethreadfence => (0, 1, Vec::new(), tcx.types.unit),
+
+        other => {
+            tcx.dcx().emit_err(UnrecognizedIntrinsicFunction { span, name: other });
+            return;
+        }
     };
     let sig = tcx.mk_fn_sig(inputs, output, false, safety, ExternAbi::Rust);
     let sig = ty::Binder::bind_with_vars(sig, bound_vars);
diff --git a/compiler/rustc_hir_analysis/src/check/wfcheck.rs b/compiler/rustc_hir_analysis/src/check/wfcheck.rs
index 2ec14b2f018..b8dc01cbc03 100644
--- a/compiler/rustc_hir_analysis/src/check/wfcheck.rs
+++ b/compiler/rustc_hir_analysis/src/check/wfcheck.rs
@@ -7,7 +7,7 @@ use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet};
 use rustc_errors::codes::*;
 use rustc_errors::{Applicability, ErrorGuaranteed, pluralize, struct_span_code_err};
 use rustc_hir::def::{DefKind, Res};
-use rustc_hir::def_id::{DefId, LocalDefId, LocalModDefId};
+use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::lang_items::LangItem;
 use rustc_hir::{AmbigArg, ItemKind};
 use rustc_infer::infer::outlives::env::OutlivesEnvironment;
@@ -40,7 +40,6 @@ use tracing::{debug, instrument};
 use {rustc_ast as ast, rustc_hir as hir};
 
 use crate::autoderef::Autoderef;
-use crate::collect::CollectItemTypesVisitor;
 use crate::constrained_generic_params::{Parameter, identify_constrained_generic_params};
 use crate::errors::InvalidReceiverTyHint;
 use crate::{errors, fluent_generated as fluent};
@@ -195,7 +194,9 @@ fn check_well_formed(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Result<(), ErrorGua
         hir::Node::TraitItem(item) => check_trait_item(tcx, item),
         hir::Node::ImplItem(item) => check_impl_item(tcx, item),
         hir::Node::ForeignItem(item) => check_foreign_item(tcx, item),
-        hir::Node::OpaqueTy(_) => Ok(crate::check::check::check_item_type(tcx, def_id)),
+        hir::Node::ConstBlock(_) | hir::Node::Expr(_) | hir::Node::OpaqueTy(_) => {
+            Ok(crate::check::check::check_item_type(tcx, def_id))
+        }
         _ => unreachable!("{node:?}"),
     };
 
@@ -229,7 +230,8 @@ fn check_item<'tcx>(tcx: TyCtxt<'tcx>, item: &'tcx hir::Item<'tcx>) -> Result<()
         ?item.owner_id,
         item.name = ? tcx.def_path_str(def_id)
     );
-    CollectItemTypesVisitor { tcx }.visit_item(item);
+    crate::collect::lower_item(tcx, item.item_id());
+    crate::collect::reject_placeholder_type_signatures_in_item(tcx, item);
 
     let res = match item.kind {
         // Right now we check that every default trait implementation
@@ -297,34 +299,30 @@ fn check_item<'tcx>(tcx: TyCtxt<'tcx>, item: &'tcx hir::Item<'tcx>) -> Result<()
         hir::ItemKind::Fn { ident, sig, .. } => {
             check_item_fn(tcx, def_id, ident, item.span, sig.decl)
         }
-        hir::ItemKind::Static(_, ty, ..) => {
-            check_item_type(tcx, def_id, ty.span, UnsizedHandling::Forbid)
-        }
-        hir::ItemKind::Const(_, ty, ..) => {
-            check_item_type(tcx, def_id, ty.span, UnsizedHandling::Forbid)
+        hir::ItemKind::Static(_, _, ty, _) => {
+            check_static_item(tcx, def_id, ty.span, UnsizedHandling::Forbid)
         }
-        hir::ItemKind::Struct(_, _, hir_generics) => {
+        hir::ItemKind::Const(_, _, ty, _) => check_const_item(tcx, def_id, ty.span, item.span),
+        hir::ItemKind::Struct(_, generics, _) => {
             let res = check_type_defn(tcx, item, false);
-            check_variances_for_type_defn(tcx, item, hir_generics);
+            check_variances_for_type_defn(tcx, item, generics);
             res
         }
-        hir::ItemKind::Union(_, _, hir_generics) => {
+        hir::ItemKind::Union(_, generics, _) => {
             let res = check_type_defn(tcx, item, true);
-            check_variances_for_type_defn(tcx, item, hir_generics);
+            check_variances_for_type_defn(tcx, item, generics);
             res
         }
-        hir::ItemKind::Enum(_, _, hir_generics) => {
+        hir::ItemKind::Enum(_, generics, _) => {
             let res = check_type_defn(tcx, item, true);
-            check_variances_for_type_defn(tcx, item, hir_generics);
+            check_variances_for_type_defn(tcx, item, generics);
             res
         }
         hir::ItemKind::Trait(..) => check_trait(tcx, item),
         hir::ItemKind::TraitAlias(..) => check_trait(tcx, item),
         // `ForeignItem`s are handled separately.
         hir::ItemKind::ForeignMod { .. } => Ok(()),
-        hir::ItemKind::TyAlias(_, hir_ty, hir_generics)
-            if tcx.type_alias_is_lazy(item.owner_id) =>
-        {
+        hir::ItemKind::TyAlias(_, generics, hir_ty) if tcx.type_alias_is_lazy(item.owner_id) => {
             let res = enter_wf_checking_ctxt(tcx, item.span, def_id, |wfcx| {
                 let ty = tcx.type_of(def_id).instantiate_identity();
                 let item_ty =
@@ -337,7 +335,7 @@ fn check_item<'tcx>(tcx: TyCtxt<'tcx>, item: &'tcx hir::Item<'tcx>) -> Result<()
                 check_where_clauses(wfcx, item.span, def_id);
                 Ok(())
             });
-            check_variances_for_type_defn(tcx, item, hir_generics);
+            check_variances_for_type_defn(tcx, item, generics);
             res
         }
         _ => Ok(()),
@@ -354,8 +352,6 @@ fn check_foreign_item<'tcx>(
 ) -> Result<(), ErrorGuaranteed> {
     let def_id = item.owner_id.def_id;
 
-    CollectItemTypesVisitor { tcx }.visit_foreign_item(item);
-
     debug!(
         ?item.owner_id,
         item.name = ? tcx.def_path_str(def_id)
@@ -366,7 +362,7 @@ fn check_foreign_item<'tcx>(
             check_item_fn(tcx, def_id, item.ident, item.span, sig.decl)
         }
         hir::ForeignItemKind::Static(ty, ..) => {
-            check_item_type(tcx, def_id, ty.span, UnsizedHandling::AllowIfForeignTail)
+            check_static_item(tcx, def_id, ty.span, UnsizedHandling::AllowIfForeignTail)
         }
         hir::ForeignItemKind::Type => Ok(()),
     }
@@ -378,7 +374,7 @@ fn check_trait_item<'tcx>(
 ) -> Result<(), ErrorGuaranteed> {
     let def_id = trait_item.owner_id.def_id;
 
-    CollectItemTypesVisitor { tcx }.visit_trait_item(trait_item);
+    crate::collect::lower_trait_item(tcx, trait_item.trait_item_id());
 
     let (method_sig, span) = match trait_item.kind {
         hir::TraitItemKind::Fn(ref sig, _) => (Some(sig), trait_item.span),
@@ -742,7 +738,7 @@ fn ty_known_to_outlive<'tcx>(
     region: ty::Region<'tcx>,
 ) -> bool {
     test_region_obligations(tcx, id, param_env, wf_tys, |infcx| {
-        infcx.register_region_obligation(infer::RegionObligation {
+        infcx.register_type_outlives_constraint_inner(infer::TypeOutlivesConstraint {
             sub_region: region,
             sup_type: ty,
             origin: infer::RelateParamBound(DUMMY_SP, ty, None),
@@ -819,7 +815,7 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for GATArgsCollector<'tcx> {
         match t.kind() {
             ty::Alias(ty::Projection, p) if p.def_id == self.gat => {
                 for (idx, arg) in p.args.iter().enumerate() {
-                    match arg.unpack() {
+                    match arg.kind() {
                         GenericArgKind::Lifetime(lt) if !lt.is_bound() => {
                             self.regions.insert((lt, idx));
                         }
@@ -943,7 +939,7 @@ fn check_impl_item<'tcx>(
     tcx: TyCtxt<'tcx>,
     impl_item: &'tcx hir::ImplItem<'tcx>,
 ) -> Result<(), ErrorGuaranteed> {
-    CollectItemTypesVisitor { tcx }.visit_impl_item(impl_item);
+    crate::collect::lower_impl_item(tcx, impl_item.impl_item_id());
 
     let (method_sig, span) = match impl_item.kind {
         hir::ImplItemKind::Fn(ref sig, _) => (Some(sig), impl_item.span),
@@ -973,7 +969,7 @@ fn check_param_wf(tcx: TyCtxt<'_>, param: &hir::GenericParam<'_>) -> Result<(),
                         ),
                         wfcx.param_env,
                         ty,
-                        tcx.require_lang_item(LangItem::UnsizedConstParamTy, Some(hir_ty.span)),
+                        tcx.require_lang_item(LangItem::UnsizedConstParamTy, hir_ty.span),
                     );
                     Ok(())
                 })
@@ -987,7 +983,7 @@ fn check_param_wf(tcx: TyCtxt<'_>, param: &hir::GenericParam<'_>) -> Result<(),
                         ),
                         wfcx.param_env,
                         ty,
-                        tcx.require_lang_item(LangItem::ConstParamTy, Some(hir_ty.span)),
+                        tcx.require_lang_item(LangItem::ConstParamTy, hir_ty.span),
                     );
                     Ok(())
                 })
@@ -1048,8 +1044,7 @@ fn check_param_wf(tcx: TyCtxt<'_>, param: &hir::GenericParam<'_>) -> Result<(),
                             match ty.kind() {
                                 ty::Adt(adt_def, ..) => adt_def.did().is_local(),
                                 // Arrays and slices use the inner type's `ConstParamTy`.
-                                ty::Array(ty, ..) => ty_is_local(*ty),
-                                ty::Slice(ty) => ty_is_local(*ty),
+                                ty::Array(ty, ..) | ty::Slice(ty) => ty_is_local(*ty),
                                 // `&` references use the inner type's `ConstParamTy`.
                                 // `&mut` are not supported.
                                 ty::Ref(_, ty, ast::Mutability::Not) => ty_is_local(*ty),
@@ -1237,7 +1232,7 @@ fn check_type_defn<'tcx>(
                     ),
                     wfcx.param_env,
                     ty,
-                    tcx.require_lang_item(LangItem::Sized, Some(hir_ty.span)),
+                    tcx.require_lang_item(LangItem::Sized, hir_ty.span),
                 );
             }
 
@@ -1331,14 +1326,13 @@ enum UnsizedHandling {
     AllowIfForeignTail,
 }
 
-fn check_item_type(
+#[instrument(level = "debug", skip(tcx, ty_span, unsized_handling))]
+fn check_static_item(
     tcx: TyCtxt<'_>,
     item_id: LocalDefId,
     ty_span: Span,
     unsized_handling: UnsizedHandling,
 ) -> Result<(), ErrorGuaranteed> {
-    debug!("check_item_type: {:?}", item_id);
-
     enter_wf_checking_ctxt(tcx, ty_span, item_id, |wfcx| {
         let ty = tcx.type_of(item_id).instantiate_identity();
         let item_ty = wfcx.deeply_normalize(ty_span, Some(WellFormedLoc::Ty(item_id)), ty);
@@ -1362,7 +1356,7 @@ fn check_item_type(
                 ),
                 wfcx.param_env,
                 item_ty,
-                tcx.require_lang_item(LangItem::Sized, Some(ty_span)),
+                tcx.require_lang_item(LangItem::Sized, ty_span),
             );
         }
 
@@ -1381,13 +1375,41 @@ fn check_item_type(
                 ),
                 wfcx.param_env,
                 item_ty,
-                tcx.require_lang_item(LangItem::Sync, Some(ty_span)),
+                tcx.require_lang_item(LangItem::Sync, ty_span),
             );
         }
         Ok(())
     })
 }
 
+fn check_const_item(
+    tcx: TyCtxt<'_>,
+    def_id: LocalDefId,
+    ty_span: Span,
+    item_span: Span,
+) -> Result<(), ErrorGuaranteed> {
+    enter_wf_checking_ctxt(tcx, ty_span, def_id, |wfcx| {
+        let ty = tcx.type_of(def_id).instantiate_identity();
+        let ty = wfcx.deeply_normalize(ty_span, Some(WellFormedLoc::Ty(def_id)), ty);
+
+        wfcx.register_wf_obligation(ty_span, Some(WellFormedLoc::Ty(def_id)), ty.into());
+        wfcx.register_bound(
+            traits::ObligationCause::new(
+                ty_span,
+                wfcx.body_def_id,
+                ObligationCauseCode::SizedConstOrStatic,
+            ),
+            wfcx.param_env,
+            ty,
+            tcx.require_lang_item(LangItem::Sized, ty_span),
+        );
+
+        check_where_clauses(wfcx, item_span, def_id);
+
+        Ok(())
+    })
+}
+
 #[instrument(level = "debug", skip(tcx, hir_self_ty, hir_trait_ref))]
 fn check_impl<'tcx>(
     tcx: TyCtxt<'tcx>,
@@ -1517,7 +1539,7 @@ fn check_where_clauses<'tcx>(wfcx: &WfCheckingCtxt<'_, 'tcx>, span: Span, def_id
             } else {
                 // If we've got a generic const parameter we still want to check its
                 // type is correct in case both it and the param type are fully concrete.
-                let GenericArgKind::Const(ct) = default.unpack() else {
+                let GenericArgKind::Const(ct) = default.kind() else {
                     continue;
                 };
 
@@ -1703,13 +1725,13 @@ fn check_fn_or_method<'tcx>(
                 ObligationCause::new(span, wfcx.body_def_id, ObligationCauseCode::RustCall),
                 wfcx.param_env,
                 *ty,
-                tcx.require_lang_item(hir::LangItem::Tuple, Some(span)),
+                tcx.require_lang_item(hir::LangItem::Tuple, span),
             );
             wfcx.register_bound(
                 ObligationCause::new(span, wfcx.body_def_id, ObligationCauseCode::RustCall),
                 wfcx.param_env,
                 *ty,
-                tcx.require_lang_item(hir::LangItem::Sized, Some(span)),
+                tcx.require_lang_item(hir::LangItem::Sized, span),
             );
         } else {
             tcx.dcx().span_err(
@@ -1754,7 +1776,7 @@ fn check_sized_if_body<'tcx>(
             ObligationCause::new(span, def_id, code),
             wfcx.param_env,
             ty,
-            tcx.require_lang_item(LangItem::Sized, Some(span)),
+            tcx.require_lang_item(LangItem::Sized, span),
         );
     }
 }
@@ -1991,7 +2013,7 @@ fn receiver_is_valid<'tcx>(
         // deref chain implement `LegacyReceiver`.
         if arbitrary_self_types_enabled.is_none() {
             let legacy_receiver_trait_def_id =
-                tcx.require_lang_item(LangItem::LegacyReceiver, Some(span));
+                tcx.require_lang_item(LangItem::LegacyReceiver, span);
             if !legacy_receiver_is_implemented(
                 wfcx,
                 legacy_receiver_trait_def_id,
@@ -2380,8 +2402,8 @@ impl<'tcx> WfCheckingCtxt<'_, 'tcx> {
     }
 }
 
-fn check_mod_type_wf(tcx: TyCtxt<'_>, module: LocalModDefId) -> Result<(), ErrorGuaranteed> {
-    let items = tcx.hir_module_items(module);
+fn check_type_wf(tcx: TyCtxt<'_>, (): ()) -> Result<(), ErrorGuaranteed> {
+    let items = tcx.hir_crate_items(());
     let res = items
         .par_items(|item| tcx.ensure_ok().check_well_formed(item.owner_id.def_id))
         .and(items.par_impl_items(|item| tcx.ensure_ok().check_well_formed(item.owner_id.def_id)))
@@ -2389,10 +2411,10 @@ fn check_mod_type_wf(tcx: TyCtxt<'_>, module: LocalModDefId) -> Result<(), Error
         .and(
             items.par_foreign_items(|item| tcx.ensure_ok().check_well_formed(item.owner_id.def_id)),
         )
+        .and(items.par_nested_bodies(|item| tcx.ensure_ok().check_well_formed(item)))
         .and(items.par_opaques(|item| tcx.ensure_ok().check_well_formed(item)));
-    if module == LocalModDefId::CRATE_DEF_ID {
-        super::entry::check_for_entry_fn(tcx);
-    }
+    super::entry::check_for_entry_fn(tcx);
+
     res
 }
 
@@ -2530,5 +2552,5 @@ struct RedundantLifetimeArgsLint<'tcx> {
 }
 
 pub fn provide(providers: &mut Providers) {
-    *providers = Providers { check_mod_type_wf, check_well_formed, ..*providers };
+    *providers = Providers { check_type_wf, check_well_formed, ..*providers };
 }
diff --git a/compiler/rustc_hir_analysis/src/coherence/builtin.rs b/compiler/rustc_hir_analysis/src/coherence/builtin.rs
index b92d1d7104f..4779f4fb702 100644
--- a/compiler/rustc_hir_analysis/src/coherence/builtin.rs
+++ b/compiler/rustc_hir_analysis/src/coherence/builtin.rs
@@ -225,7 +225,7 @@ fn visit_implementation_of_dispatch_from_dyn(checker: &Checker<'_>) -> Result<()
     // redundant errors for `DispatchFromDyn`. This is best effort, though.
     let mut res = Ok(());
     tcx.for_each_relevant_impl(
-        tcx.require_lang_item(LangItem::CoerceUnsized, Some(span)),
+        tcx.require_lang_item(LangItem::CoerceUnsized, span),
         source,
         |impl_def_id| {
             res = res.and(tcx.ensure_ok().coerce_unsized_info(impl_def_id));
@@ -379,8 +379,8 @@ pub(crate) fn coerce_unsized_info<'tcx>(
     let span = tcx.def_span(impl_did);
     let trait_name = "CoerceUnsized";
 
-    let coerce_unsized_trait = tcx.require_lang_item(LangItem::CoerceUnsized, Some(span));
-    let unsize_trait = tcx.require_lang_item(LangItem::Unsize, Some(span));
+    let coerce_unsized_trait = tcx.require_lang_item(LangItem::CoerceUnsized, span);
+    let unsize_trait = tcx.require_lang_item(LangItem::Unsize, span);
 
     let source = tcx.type_of(impl_did).instantiate_identity();
     let trait_ref = tcx.impl_trait_ref(impl_did).unwrap().instantiate_identity();
@@ -591,7 +591,7 @@ fn infringing_fields_error<'tcx>(
     impl_did: LocalDefId,
     impl_span: Span,
 ) -> ErrorGuaranteed {
-    let trait_did = tcx.require_lang_item(lang_item, Some(impl_span));
+    let trait_did = tcx.require_lang_item(lang_item, impl_span);
 
     let trait_name = tcx.def_path_str(trait_did);
 
@@ -748,7 +748,7 @@ fn visit_implementation_of_pointer_like(checker: &Checker<'_>) -> Result<(), Err
                         ObligationCause::misc(impl_span, checker.impl_def_id),
                         param_env,
                         nontrivial_field_ty,
-                        tcx.require_lang_item(LangItem::PointerLike, Some(impl_span)),
+                        tcx.require_lang_item(LangItem::PointerLike, impl_span),
                     );
                     // FIXME(dyn-star): We should regionck this implementation.
                     if ocx.select_all_or_error().is_empty() {
diff --git a/compiler/rustc_hir_analysis/src/collect.rs b/compiler/rustc_hir_analysis/src/collect.rs
index 4520fbe352c..e1df0d60452 100644
--- a/compiler/rustc_hir_analysis/src/collect.rs
+++ b/compiler/rustc_hir_analysis/src/collect.rs
@@ -28,11 +28,10 @@ use rustc_errors::{
 };
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LocalDefId};
-use rustc_hir::intravisit::{self, InferKind, Visitor, VisitorExt, walk_generics};
+use rustc_hir::intravisit::{InferKind, Visitor, VisitorExt, walk_generics};
 use rustc_hir::{self as hir, GenericParamKind, HirId, Node, PreciseCapturingArgKind};
 use rustc_infer::infer::{InferCtxt, TyCtxtInferExt};
-use rustc_infer::traits::ObligationCause;
-use rustc_middle::hir::nested_filter;
+use rustc_infer::traits::{DynCompatibilityViolation, ObligationCause};
 use rustc_middle::query::Providers;
 use rustc_middle::ty::util::{Discr, IntTypeExt};
 use rustc_middle::ty::{self, AdtKind, Const, IsSuggestable, Ty, TyCtxt, TypingMode, fold_regions};
@@ -40,11 +39,10 @@ use rustc_middle::{bug, span_bug};
 use rustc_span::{DUMMY_SP, Ident, Span, Symbol, kw, sym};
 use rustc_trait_selection::error_reporting::traits::suggestions::NextTypeParamName;
 use rustc_trait_selection::infer::InferCtxtExt;
-use rustc_trait_selection::traits::ObligationCtxt;
+use rustc_trait_selection::traits::{ObligationCtxt, hir_ty_lowering_dyn_compatibility_violations};
 use tracing::{debug, instrument};
 
 use crate::errors;
-use crate::hir_ty_lowering::errors::assoc_tag_str;
 use crate::hir_ty_lowering::{FeedConstTy, HirTyLowerer, RegionInferReason};
 
 pub(crate) mod dump;
@@ -89,6 +87,7 @@ pub(crate) fn provide(providers: &mut Providers) {
         opaque_ty_origin,
         rendered_precise_capturing_args,
         const_param_default,
+        anon_const_kind,
         ..*providers
     };
 }
@@ -148,10 +147,6 @@ impl<'v> Visitor<'v> for HirPlaceholderCollector {
     }
 }
 
-pub(crate) struct CollectItemTypesVisitor<'tcx> {
-    pub tcx: TyCtxt<'tcx>,
-}
-
 /// If there are any placeholder types (`_`), emit an error explaining that this is not allowed
 /// and suggest adding type parameters in the appropriate place, taking into consideration any and
 /// all already existing generic type parameters to avoid suggesting a name that is already in use.
@@ -243,18 +238,18 @@ pub(crate) fn placeholder_type_error_diag<'cx, 'tcx>(
     err
 }
 
-fn reject_placeholder_type_signatures_in_item<'tcx>(
+pub(super) fn reject_placeholder_type_signatures_in_item<'tcx>(
     tcx: TyCtxt<'tcx>,
     item: &'tcx hir::Item<'tcx>,
 ) {
     let (generics, suggest) = match &item.kind {
-        hir::ItemKind::Union(_, _, generics)
-        | hir::ItemKind::Enum(_, _, generics)
+        hir::ItemKind::Union(_, generics, _)
+        | hir::ItemKind::Enum(_, generics, _)
         | hir::ItemKind::TraitAlias(_, generics, _)
         | hir::ItemKind::Trait(_, _, _, generics, ..)
         | hir::ItemKind::Impl(hir::Impl { generics, .. })
-        | hir::ItemKind::Struct(_, _, generics) => (generics, true),
-        hir::ItemKind::TyAlias(_, _, generics) => (generics, false),
+        | hir::ItemKind::Struct(_, generics, _) => (generics, true),
+        hir::ItemKind::TyAlias(_, generics, _) => (generics, false),
         // `static`, `fn` and `const` are handled elsewhere to suggest appropriate type.
         _ => return,
     };
@@ -274,81 +269,6 @@ fn reject_placeholder_type_signatures_in_item<'tcx>(
     );
 }
 
-impl<'tcx> Visitor<'tcx> for CollectItemTypesVisitor<'tcx> {
-    type NestedFilter = nested_filter::OnlyBodies;
-
-    fn maybe_tcx(&mut self) -> Self::MaybeTyCtxt {
-        self.tcx
-    }
-
-    fn visit_item(&mut self, item: &'tcx hir::Item<'tcx>) {
-        lower_item(self.tcx, item.item_id());
-        reject_placeholder_type_signatures_in_item(self.tcx, item);
-        intravisit::walk_item(self, item);
-    }
-
-    fn visit_generics(&mut self, generics: &'tcx hir::Generics<'tcx>) {
-        for param in generics.params {
-            match param.kind {
-                hir::GenericParamKind::Lifetime { .. } => {}
-                hir::GenericParamKind::Type { default: Some(_), .. } => {
-                    self.tcx.ensure_ok().type_of(param.def_id);
-                }
-                hir::GenericParamKind::Type { .. } => {}
-                hir::GenericParamKind::Const { default, .. } => {
-                    self.tcx.ensure_ok().type_of(param.def_id);
-                    if let Some(default) = default {
-                        // need to store default and type of default
-                        self.tcx.ensure_ok().const_param_default(param.def_id);
-                        if let hir::ConstArgKind::Anon(ac) = default.kind {
-                            self.tcx.ensure_ok().type_of(ac.def_id);
-                        }
-                    }
-                }
-            }
-        }
-        intravisit::walk_generics(self, generics);
-    }
-
-    fn visit_expr(&mut self, expr: &'tcx hir::Expr<'tcx>) {
-        if let hir::ExprKind::Closure(closure) = expr.kind {
-            self.tcx.ensure_ok().generics_of(closure.def_id);
-            self.tcx.ensure_ok().codegen_fn_attrs(closure.def_id);
-            // We do not call `type_of` for closures here as that
-            // depends on typecheck and would therefore hide
-            // any further errors in case one typeck fails.
-        }
-        intravisit::walk_expr(self, expr);
-    }
-
-    /// Don't call `type_of` on opaque types, since that depends on type checking function bodies.
-    /// `check_item_type` ensures that it's called instead.
-    fn visit_opaque_ty(&mut self, opaque: &'tcx hir::OpaqueTy<'tcx>) {
-        let def_id = opaque.def_id;
-        self.tcx.ensure_ok().generics_of(def_id);
-        self.tcx.ensure_ok().predicates_of(def_id);
-        self.tcx.ensure_ok().explicit_item_bounds(def_id);
-        self.tcx.ensure_ok().explicit_item_self_bounds(def_id);
-        self.tcx.ensure_ok().item_bounds(def_id);
-        self.tcx.ensure_ok().item_self_bounds(def_id);
-        if self.tcx.is_conditionally_const(def_id) {
-            self.tcx.ensure_ok().explicit_implied_const_bounds(def_id);
-            self.tcx.ensure_ok().const_conditions(def_id);
-        }
-        intravisit::walk_opaque_ty(self, opaque);
-    }
-
-    fn visit_trait_item(&mut self, trait_item: &'tcx hir::TraitItem<'tcx>) {
-        lower_trait_item(self.tcx, trait_item.trait_item_id());
-        intravisit::walk_trait_item(self, trait_item);
-    }
-
-    fn visit_impl_item(&mut self, impl_item: &'tcx hir::ImplItem<'tcx>) {
-        lower_impl_item(self.tcx, impl_item.impl_item_id());
-        intravisit::walk_impl_item(self, impl_item);
-    }
-}
-
 ///////////////////////////////////////////////////////////////////////////
 // Utility types and common code for the above passes.
 
@@ -444,13 +364,12 @@ impl<'tcx> HirTyLowerer<'tcx> for ItemCtxt<'tcx> {
         self.tcx.at(span).type_param_predicates((self.item_def_id, def_id, assoc_ident))
     }
 
-    fn lower_assoc_shared(
+    fn lower_assoc_item_path(
         &self,
         span: Span,
         item_def_id: DefId,
         item_segment: &rustc_hir::PathSegment<'tcx>,
         poly_trait_ref: ty::PolyTraitRef<'tcx>,
-        assoc_tag: ty::AssocTag,
     ) -> Result<(DefId, ty::GenericArgsRef<'tcx>), ErrorGuaranteed> {
         if let Some(trait_ref) = poly_trait_ref.no_bound_vars() {
             let item_args = self.lowerer().lower_generic_args_of_assoc_item(
@@ -471,9 +390,9 @@ impl<'tcx> HirTyLowerer<'tcx> for ItemCtxt<'tcx> {
                         .tcx
                         .hir_expect_item(self.tcx.hir_get_parent_item(self.hir_id()).def_id);
                     match &item.kind {
-                        hir::ItemKind::Enum(_, _, generics)
-                        | hir::ItemKind::Struct(_, _, generics)
-                        | hir::ItemKind::Union(_, _, generics) => {
+                        hir::ItemKind::Enum(_, generics, _)
+                        | hir::ItemKind::Struct(_, generics, _)
+                        | hir::ItemKind::Union(_, generics, _) => {
                             let lt_name = get_new_lifetime_name(self.tcx, poly_trait_ref, generics);
                             let (lt_sp, sugg) = match generics.params {
                                 [] => (generics.span, format!("<{lt_name}>")),
@@ -525,7 +444,7 @@ impl<'tcx> HirTyLowerer<'tcx> for ItemCtxt<'tcx> {
                 inferred_sugg,
                 bound,
                 mpart_sugg,
-                what: assoc_tag_str(assoc_tag),
+                what: self.tcx.def_descr(item_def_id),
             }))
         }
     }
@@ -626,6 +545,10 @@ impl<'tcx> HirTyLowerer<'tcx> for ItemCtxt<'tcx> {
 
         (input_tys, output_ty)
     }
+
+    fn dyn_compatibility_violations(&self, trait_def_id: DefId) -> Vec<DynCompatibilityViolation> {
+        hir_ty_lowering_dyn_compatibility_violations(self.tcx, trait_def_id)
+    }
 }
 
 /// Synthesize a new lifetime name that doesn't clash with any of the lifetimes already present.
@@ -666,7 +589,7 @@ fn get_new_lifetime_name<'tcx>(
 }
 
 #[instrument(level = "debug", skip_all)]
-fn lower_item(tcx: TyCtxt<'_>, item_id: hir::ItemId) {
+pub(super) fn lower_item(tcx: TyCtxt<'_>, item_id: hir::ItemId) {
     let it = tcx.hir_item(item_id);
     debug!(item = ?it.kind.ident(), id = %it.hir_id());
     let def_id = item_id.owner_id.def_id;
@@ -737,7 +660,7 @@ fn lower_item(tcx: TyCtxt<'_>, item_id: hir::ItemId) {
             tcx.at(it.span).explicit_super_predicates_of(def_id);
             tcx.ensure_ok().predicates_of(def_id);
         }
-        hir::ItemKind::Struct(_, struct_def, _) | hir::ItemKind::Union(_, struct_def, _) => {
+        hir::ItemKind::Struct(_, _, struct_def) | hir::ItemKind::Union(_, _, struct_def) => {
             tcx.ensure_ok().generics_of(def_id);
             tcx.ensure_ok().type_of(def_id);
             tcx.ensure_ok().predicates_of(def_id);
@@ -759,7 +682,7 @@ fn lower_item(tcx: TyCtxt<'_>, item_id: hir::ItemId) {
             tcx.ensure_ok().predicates_of(def_id);
         }
 
-        hir::ItemKind::Static(_, ty, ..) | hir::ItemKind::Const(_, ty, ..) => {
+        hir::ItemKind::Static(_, _, ty, _) | hir::ItemKind::Const(_, _, ty, _) => {
             tcx.ensure_ok().generics_of(def_id);
             tcx.ensure_ok().type_of(def_id);
             tcx.ensure_ok().predicates_of(def_id);
@@ -787,7 +710,7 @@ fn lower_item(tcx: TyCtxt<'_>, item_id: hir::ItemId) {
     }
 }
 
-fn lower_trait_item(tcx: TyCtxt<'_>, trait_item_id: hir::TraitItemId) {
+pub(crate) fn lower_trait_item(tcx: TyCtxt<'_>, trait_item_id: hir::TraitItemId) {
     let trait_item = tcx.hir_trait_item(trait_item_id);
     let def_id = trait_item_id.owner_id;
     tcx.ensure_ok().generics_of(def_id);
@@ -858,7 +781,7 @@ fn lower_trait_item(tcx: TyCtxt<'_>, trait_item_id: hir::TraitItemId) {
     tcx.ensure_ok().predicates_of(def_id);
 }
 
-fn lower_impl_item(tcx: TyCtxt<'_>, impl_item_id: hir::ImplItemId) {
+pub(super) fn lower_impl_item(tcx: TyCtxt<'_>, impl_item_id: hir::ImplItemId) {
     let def_id = impl_item_id.owner_id;
     tcx.ensure_ok().generics_of(def_id);
     tcx.ensure_ok().type_of(def_id);
@@ -1090,7 +1013,7 @@ fn adt_def(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::AdtDef<'_> {
 
     let repr = tcx.repr_options_of_def(def_id);
     let (kind, variants) = match &item.kind {
-        ItemKind::Enum(_, def, _) => {
+        ItemKind::Enum(_, _, def) => {
             let mut distance_from_explicit = 0;
             let variants = def
                 .variants
@@ -1118,7 +1041,7 @@ fn adt_def(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::AdtDef<'_> {
 
             (AdtKind::Enum, variants)
         }
-        ItemKind::Struct(ident, def, _) | ItemKind::Union(ident, def, _) => {
+        ItemKind::Struct(ident, _, def) | ItemKind::Union(ident, _, def) => {
             let adt_kind = match item.kind {
                 ItemKind::Struct(..) => AdtKind::Struct,
                 _ => AdtKind::Union,
@@ -1828,3 +1751,27 @@ fn const_param_default<'tcx>(
         .lower_const_arg(default_ct, FeedConstTy::Param(def_id.to_def_id(), identity_args));
     ty::EarlyBinder::bind(ct)
 }
+
+fn anon_const_kind<'tcx>(tcx: TyCtxt<'tcx>, def: LocalDefId) -> ty::AnonConstKind {
+    let hir_id = tcx.local_def_id_to_hir_id(def);
+    let const_arg_id = tcx.parent_hir_id(hir_id);
+    match tcx.hir_node(const_arg_id) {
+        hir::Node::ConstArg(_) => {
+            if tcx.features().generic_const_exprs() {
+                ty::AnonConstKind::GCE
+            } else if tcx.features().min_generic_const_args() {
+                ty::AnonConstKind::MCG
+            } else if let hir::Node::Expr(hir::Expr {
+                kind: hir::ExprKind::Repeat(_, repeat_count),
+                ..
+            }) = tcx.hir_node(tcx.parent_hir_id(const_arg_id))
+                && repeat_count.hir_id == const_arg_id
+            {
+                ty::AnonConstKind::RepeatExprCount
+            } else {
+                ty::AnonConstKind::MCG
+            }
+        }
+        _ => ty::AnonConstKind::NonTypeSystem,
+    }
+}
diff --git a/compiler/rustc_hir_analysis/src/collect/generics_of.rs b/compiler/rustc_hir_analysis/src/collect/generics_of.rs
index 2bed28d7b71..7eb896f0bf1 100644
--- a/compiler/rustc_hir_analysis/src/collect/generics_of.rs
+++ b/compiler/rustc_hir_analysis/src/collect/generics_of.rs
@@ -104,19 +104,27 @@ pub(super) fn generics_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Generics {
                 }
             }
 
-            if in_param_ty {
-                // We do not allow generic parameters in anon consts if we are inside
-                // of a const parameter type, e.g. `struct Foo<const N: usize, const M: [u8; N]>` is not allowed.
-                None
-            } else if tcx.features().generic_const_exprs() {
-                let parent_node = tcx.parent_hir_node(hir_id);
-                debug!(?parent_node);
-                if let Node::Variant(Variant { disr_expr: Some(constant), .. }) = parent_node
-                    && constant.hir_id == hir_id
-                {
-                    // enum variant discriminants are not allowed to use any kind of generics
-                    None
-                } else if let Some(param_id) = tcx.hir_opt_const_param_default_param_def_id(hir_id)
+            match tcx.anon_const_kind(def_id) {
+                // Stable: anon consts are not able to use any generic parameters...
+                ty::AnonConstKind::MCG => None,
+                // we provide generics to repeat expr counts as a backwards compatibility hack. #76200
+                ty::AnonConstKind::RepeatExprCount => Some(parent_did),
+
+                // Even GCE anon const should not be allowed to use generic parameters as it would be
+                // trivially forward declared uses once desugared. E.g. `const N: [u8; ANON::<N>]`.
+                //
+                // We could potentially mirror the hack done for defaults of generic parameters but
+                // this case just doesn't come up much compared to `const N: u32 = ...`. Long term the
+                // hack for defaulted parameters should be removed eventually anyway.
+                ty::AnonConstKind::GCE if in_param_ty => None,
+                // GCE anon consts as a default for a generic parameter should have their provided generics
+                // "truncated" up to whatever generic parameter this anon const is within the default of.
+                //
+                // FIXME(generic_const_exprs): This only handles `const N: usize = /*defid*/` but not type
+                // parameter defaults, e.g. `T = Foo</*defid*/>`.
+                ty::AnonConstKind::GCE
+                    if let Some(param_id) =
+                        tcx.hir_opt_const_param_default_param_def_id(hir_id) =>
                 {
                     // If the def_id we are calling generics_of on is an anon ct default i.e:
                     //
@@ -160,36 +168,17 @@ pub(super) fn generics_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Generics {
                         has_self: generics.has_self,
                         has_late_bound_regions: generics.has_late_bound_regions,
                     };
-                } else {
-                    // HACK(eddyb) this provides the correct generics when
-                    // `feature(generic_const_expressions)` is enabled, so that const expressions
-                    // used with const generics, e.g. `Foo<{N+1}>`, can work at all.
-                    //
-                    // Note that we do not supply the parent generics when using
-                    // `min_const_generics`.
-                    Some(parent_did)
                 }
-            } else {
-                let parent_node = tcx.parent_hir_node(hir_id);
-                let parent_node = match parent_node {
-                    Node::ConstArg(ca) => tcx.parent_hir_node(ca.hir_id),
-                    _ => parent_node,
-                };
-                match parent_node {
-                    // HACK(eddyb) this provides the correct generics for repeat
-                    // expressions' count (i.e. `N` in `[x; N]`), and explicit
-                    // `enum` discriminants (i.e. `D` in `enum Foo { Bar = D }`),
-                    // as they shouldn't be able to cause query cycle errors.
-                    Node::Expr(Expr { kind: ExprKind::Repeat(_, ct), .. })
-                        if ct.anon_const_hir_id() == Some(hir_id) =>
-                    {
-                        Some(parent_did)
-                    }
-                    Node::TyPat(_) => Some(parent_did),
-                    // Field default values inherit the ADT's generics.
-                    Node::Field(_) => Some(parent_did),
-                    _ => None,
+                ty::AnonConstKind::GCE => Some(parent_did),
+
+                // Field defaults are allowed to use generic parameters, e.g. `field: u32 = /*defid: N + 1*/`
+                ty::AnonConstKind::NonTypeSystem
+                    if matches!(tcx.parent_hir_node(hir_id), Node::TyPat(_) | Node::Field(_)) =>
+                {
+                    Some(parent_did)
                 }
+                // Default to no generic parameters for other kinds of anon consts
+                ty::AnonConstKind::NonTypeSystem => None,
             }
         }
         Node::ConstBlock(_)
diff --git a/compiler/rustc_hir_analysis/src/collect/item_bounds.rs b/compiler/rustc_hir_analysis/src/collect/item_bounds.rs
index 279b1e82a71..5f1cdeddc19 100644
--- a/compiler/rustc_hir_analysis/src/collect/item_bounds.rs
+++ b/compiler/rustc_hir_analysis/src/collect/item_bounds.rs
@@ -151,7 +151,7 @@ fn remap_gat_vars_and_recurse_into_nested_projections<'tcx>(
     let mut mapping = FxIndexMap::default();
     let generics = tcx.generics_of(assoc_item_def_id);
     for (param, var) in std::iter::zip(&generics.own_params, gat_vars) {
-        let existing = match var.unpack() {
+        let existing = match var.kind() {
             ty::GenericArgKind::Lifetime(re) => {
                 if let ty::RegionKind::ReBound(ty::INNERMOST, bv) = re.kind() {
                     mapping.insert(bv.var, tcx.mk_param_from_def(param))
diff --git a/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs b/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs
index 709446d09cd..d45f0475e99 100644
--- a/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs
+++ b/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs
@@ -634,11 +634,11 @@ impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> {
                 // These sorts of items have no lifetime parameters at all.
                 intravisit::walk_item(self, item);
             }
-            hir::ItemKind::TyAlias(_, _, generics)
-            | hir::ItemKind::Const(_, _, generics, _)
-            | hir::ItemKind::Enum(_, _, generics)
-            | hir::ItemKind::Struct(_, _, generics)
-            | hir::ItemKind::Union(_, _, generics)
+            hir::ItemKind::TyAlias(_, generics, _)
+            | hir::ItemKind::Const(_, generics, _, _)
+            | hir::ItemKind::Enum(_, generics, _)
+            | hir::ItemKind::Struct(_, generics, _)
+            | hir::ItemKind::Union(_, generics, _)
             | hir::ItemKind::Trait(_, _, _, generics, ..)
             | hir::ItemKind::TraitAlias(_, generics, ..)
             | hir::ItemKind::Impl(&hir::Impl { generics, .. }) => {
diff --git a/compiler/rustc_hir_analysis/src/collect/type_of.rs b/compiler/rustc_hir_analysis/src/collect/type_of.rs
index c20b14df770..141d96b57e5 100644
--- a/compiler/rustc_hir_analysis/src/collect/type_of.rs
+++ b/compiler/rustc_hir_analysis/src/collect/type_of.rs
@@ -206,7 +206,7 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::EarlyBinder<'_
         },
 
         Node::Item(item) => match item.kind {
-            ItemKind::Static(ident, ty, .., body_id) => {
+            ItemKind::Static(_, ident, ty, body_id) => {
                 if ty.is_suggestable_infer_ty() {
                     infer_placeholder_type(
                         icx.lowerer(),
@@ -220,7 +220,7 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::EarlyBinder<'_
                     icx.lower_ty(ty)
                 }
             }
-            ItemKind::Const(ident, ty, _, body_id) => {
+            ItemKind::Const(ident, _, ty, body_id) => {
                 if ty.is_suggestable_infer_ty() {
                     infer_placeholder_type(
                         icx.lowerer(),
@@ -234,7 +234,7 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::EarlyBinder<'_
                     icx.lower_ty(ty)
                 }
             }
-            ItemKind::TyAlias(_, self_ty, _) => icx.lower_ty(self_ty),
+            ItemKind::TyAlias(_, _, self_ty) => icx.lower_ty(self_ty),
             ItemKind::Impl(hir::Impl { self_ty, .. }) => match self_ty.find_self_aliases() {
                 spans if spans.len() > 0 => {
                     let guar = tcx
@@ -532,5 +532,5 @@ pub(crate) fn type_alias_is_lazy<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) ->
             }
         }
     }
-    HasTait.visit_ty_unambig(tcx.hir_expect_item(def_id).expect_ty_alias().1).is_break()
+    HasTait.visit_ty_unambig(tcx.hir_expect_item(def_id).expect_ty_alias().2).is_break()
 }
diff --git a/compiler/rustc_hir_analysis/src/errors.rs b/compiler/rustc_hir_analysis/src/errors.rs
index 152714b3407..a27d1ed6c53 100644
--- a/compiler/rustc_hir_analysis/src/errors.rs
+++ b/compiler/rustc_hir_analysis/src/errors.rs
@@ -162,15 +162,6 @@ pub(crate) enum AssocItemNotFoundSugg<'a> {
 }
 
 #[derive(Diagnostic)]
-#[diag(hir_analysis_unrecognized_atomic_operation, code = E0092)]
-pub(crate) struct UnrecognizedAtomicOperation<'a> {
-    #[primary_span]
-    #[label]
-    pub span: Span,
-    pub op: &'a str,
-}
-
-#[derive(Diagnostic)]
 #[diag(hir_analysis_wrong_number_of_generic_arguments_to_intrinsic, code = E0094)]
 pub(crate) struct WrongNumberOfGenericArgumentsToIntrinsic<'a> {
     #[primary_span]
diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/bounds.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/bounds.rs
index 4419d5dc7d6..bdc42c7a2d9 100644
--- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/bounds.rs
+++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/bounds.rs
@@ -4,9 +4,9 @@ use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_errors::codes::*;
 use rustc_errors::struct_span_code_err;
 use rustc_hir as hir;
+use rustc_hir::AmbigArg;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::{DefId, LocalDefId};
-use rustc_hir::{AmbigArg, HirId};
 use rustc_middle::bug;
 use rustc_middle::ty::{
     self as ty, IsSuggestable, Ty, TyCtxt, TypeFoldable, TypeSuperVisitable, TypeVisitable,
@@ -309,7 +309,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
                     false => "`?Sized`",
                 };
                 // There was a `?Trait` bound, but it was neither `?Sized` nor `experimental_default_bounds`.
-                tcx.dcx().span_err(
+                self.dcx().span_err(
                     unbound.span,
                     format!(
                         "relaxing a default bound only does something for {}; \
@@ -675,7 +675,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
 
                 // Good error for `where Trait::method(..): Send`.
                 let Some(self_ty) = opt_self_ty else {
-                    let guar = self.error_missing_qpath_self_ty(
+                    let guar = self.report_missing_self_ty_for_resolved_path(
                         trait_def_id,
                         hir_ty.span,
                         item_segment,
@@ -713,120 +713,51 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
                     Err(guar) => Ty::new_error(tcx, guar),
                 }
             }
-            hir::QPath::TypeRelative(qself, item_segment)
-                if item_segment.args.is_some_and(|args| {
+            hir::QPath::TypeRelative(hir_self_ty, segment)
+                if segment.args.is_some_and(|args| {
                     matches!(args.parenthesized, hir::GenericArgsParentheses::ReturnTypeNotation)
                 }) =>
             {
-                match self
-                    .resolve_type_relative_return_type_notation(
-                        qself,
-                        item_segment,
-                        hir_ty.hir_id,
-                        hir_ty.span,
-                    )
-                    .and_then(|(candidate, item_def_id)| {
-                        self.lower_return_type_notation_ty(candidate, item_def_id, hir_ty.span)
-                    }) {
-                    Ok(ty) => Ty::new_alias(tcx, ty::Projection, ty),
-                    Err(guar) => Ty::new_error(tcx, guar),
-                }
-            }
-            _ => self.lower_ty(hir_ty),
-        }
-    }
-
-    /// Perform type-dependent lookup for a *method* for return type notation.
-    /// This generally mirrors `<dyn HirTyLowerer>::lower_assoc_path`.
-    fn resolve_type_relative_return_type_notation(
-        &self,
-        qself: &'tcx hir::Ty<'tcx>,
-        item_segment: &'tcx hir::PathSegment<'tcx>,
-        qpath_hir_id: HirId,
-        span: Span,
-    ) -> Result<(ty::PolyTraitRef<'tcx>, DefId), ErrorGuaranteed> {
-        let tcx = self.tcx();
-        let qself_ty = self.lower_ty(qself);
-        let assoc_ident = item_segment.ident;
-        let qself_res = if let hir::TyKind::Path(hir::QPath::Resolved(_, path)) = &qself.kind {
-            path.res
-        } else {
-            Res::Err
-        };
-
-        let bound = match (qself_ty.kind(), qself_res) {
-            (_, Res::SelfTyAlias { alias_to: impl_def_id, is_trait_impl: true, .. }) => {
-                // `Self` in an impl of a trait -- we have a concrete self type and a
-                // trait reference.
-                let Some(trait_ref) = tcx.impl_trait_ref(impl_def_id) else {
-                    // A cycle error occurred, most likely.
-                    self.dcx().span_bug(span, "expected cycle error");
-                };
-
-                self.probe_single_bound_for_assoc_item(
-                    || {
-                        traits::supertraits(
-                            tcx,
-                            ty::Binder::dummy(trait_ref.instantiate_identity()),
-                        )
-                    },
-                    AssocItemQSelf::SelfTyAlias,
+                let self_ty = self.lower_ty(hir_self_ty);
+                let (item_def_id, bound) = match self.resolve_type_relative_path(
+                    self_ty,
+                    hir_self_ty,
                     ty::AssocTag::Fn,
-                    assoc_ident,
-                    span,
+                    segment,
+                    hir_ty.hir_id,
+                    hir_ty.span,
                     None,
-                )?
-            }
-            (
-                &ty::Param(_),
-                Res::SelfTyParam { trait_: param_did } | Res::Def(DefKind::TyParam, param_did),
-            ) => self.probe_single_ty_param_bound_for_assoc_item(
-                param_did.expect_local(),
-                qself.span,
-                ty::AssocTag::Fn,
-                assoc_ident,
-                span,
-            )?,
-            _ => {
-                if let Err(reported) = qself_ty.error_reported() {
-                    return Err(reported);
-                } else {
-                    // FIXME(return_type_notation): Provide some structured suggestion here.
-                    let err = struct_span_code_err!(
-                        self.dcx(),
-                        span,
-                        E0223,
-                        "ambiguous associated function"
+                ) {
+                    Ok(result) => result,
+                    Err(guar) => return Ty::new_error(tcx, guar),
+                };
+
+                // Don't let `T::method` resolve to some `for<'a> <T as Tr<'a>>::method`,
+                // which may happen via a higher-ranked where clause or supertrait.
+                // This is the same restrictions as associated types; even though we could
+                // support it, it just makes things a lot more difficult to support in
+                // `resolve_bound_vars`, since we'd need to introduce those as elided
+                // bound vars on the where clause too.
+                if bound.has_bound_vars() {
+                    return Ty::new_error(
+                        tcx,
+                        self.dcx().emit_err(errors::AssociatedItemTraitUninferredGenericParams {
+                            span: hir_ty.span,
+                            inferred_sugg: Some(hir_ty.span.with_hi(segment.ident.span.lo())),
+                            bound: format!("{}::", tcx.anonymize_bound_vars(bound).skip_binder()),
+                            mpart_sugg: None,
+                            what: tcx.def_descr(item_def_id),
+                        }),
                     );
-                    return Err(err.emit());
                 }
-            }
-        };
 
-        // Don't let `T::method` resolve to some `for<'a> <T as Tr<'a>>::method`,
-        // which may happen via a higher-ranked where clause or supertrait.
-        // This is the same restrictions as associated types; even though we could
-        // support it, it just makes things a lot more difficult to support in
-        // `resolve_bound_vars`, since we'd need to introduce those as elided
-        // bound vars on the where clause too.
-        if bound.has_bound_vars() {
-            return Err(self.tcx().dcx().emit_err(
-                errors::AssociatedItemTraitUninferredGenericParams {
-                    span,
-                    inferred_sugg: Some(span.with_hi(item_segment.ident.span.lo())),
-                    bound: format!("{}::", tcx.anonymize_bound_vars(bound).skip_binder(),),
-                    mpart_sugg: None,
-                    what: "function",
-                },
-            ));
+                match self.lower_return_type_notation_ty(bound, item_def_id, hir_ty.span) {
+                    Ok(ty) => Ty::new_alias(tcx, ty::Projection, ty),
+                    Err(guar) => Ty::new_error(tcx, guar),
+                }
+            }
+            _ => self.lower_ty(hir_ty),
         }
-
-        let trait_def_id = bound.def_id();
-        let assoc_ty = self
-            .probe_assoc_item(assoc_ident, ty::AssocTag::Fn, qpath_hir_id, span, trait_def_id)
-            .expect("failed to find associated type");
-
-        Ok((bound, assoc_ty.def_id))
     }
 
     /// Do the common parts of lowering an RTN type. This involves extending the
diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/cmse.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/cmse.rs
index d1ee5a5494c..ebeb3b58208 100644
--- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/cmse.rs
+++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/cmse.rs
@@ -35,7 +35,7 @@ pub(crate) fn validate_cmse_abi<'tcx>(
                     _ => tcx.hir_span(hir_id),
                 };
                 struct_span_code_err!(
-                    tcx.dcx(),
+                    dcx,
                     span,
                     E0781,
                     "the `\"C-cmse-nonsecure-call\"` ABI is only allowed on function pointers"
diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/dyn_compatibility.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/dyn_compatibility.rs
index f6e5149bd2b..c7cdf1d5bd2 100644
--- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/dyn_compatibility.rs
+++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/dyn_compatibility.rs
@@ -11,7 +11,7 @@ use rustc_middle::ty::{
 };
 use rustc_span::{ErrorGuaranteed, Span};
 use rustc_trait_selection::error_reporting::traits::report_dyn_incompatibility;
-use rustc_trait_selection::traits::{self, hir_ty_lowering_dyn_compatibility_violations};
+use rustc_trait_selection::traits;
 use smallvec::{SmallVec, smallvec};
 use tracing::{debug, instrument};
 
@@ -78,15 +78,13 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
 
         // We  don't support empty trait objects.
         if regular_traits.is_empty() && auto_traits.is_empty() {
-            let guar = self.report_trait_object_with_no_traits_error(
-                span,
-                user_written_bounds.iter().copied(),
-            );
+            let guar =
+                self.report_trait_object_with_no_traits(span, user_written_bounds.iter().copied());
             return Ty::new_error(tcx, guar);
         }
         // We don't support >1 principal
         if regular_traits.len() > 1 {
-            let guar = self.report_trait_object_addition_traits_error(&regular_traits);
+            let guar = self.report_trait_object_addition_traits(&regular_traits);
             return Ty::new_error(tcx, guar);
         }
         // Don't create a dyn trait if we have errors in the principal.
@@ -99,8 +97,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         // to avoid ICEs.
         for (clause, span) in user_written_bounds {
             if let Some(trait_pred) = clause.as_trait_clause() {
-                let violations =
-                    hir_ty_lowering_dyn_compatibility_violations(tcx, trait_pred.def_id());
+                let violations = self.dyn_compatibility_violations(trait_pred.def_id());
                 if !violations.is_empty() {
                     let reported = report_dyn_incompatibility(
                         tcx,
@@ -132,7 +129,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
                     if references_self {
                         // With trait alias and type alias combined, type resolver
                         // may not be able to catch all illegal `Self` usages (issue 139082)
-                        let guar = tcx.dcx().emit_err(SelfInTypeAlias { span });
+                        let guar = self.dcx().emit_err(SelfInTypeAlias { span });
                         b.term = replace_dummy_self_with_error(tcx, b.term, guar);
                     }
                 }
@@ -218,7 +215,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
                         let pred = bound_predicate.rebind(pred);
                         // A `Self` within the original bound will be instantiated with a
                         // `trait_object_dummy_self`, so check for that.
-                        let references_self = match pred.skip_binder().term.unpack() {
+                        let references_self = match pred.skip_binder().term.kind() {
                             ty::TermKind::Ty(ty) => ty.walk().any(|arg| arg == dummy_self.into()),
                             // FIXME(associated_const_equality): We should walk the const instead of not doing anything
                             ty::TermKind::Const(_) => false,
@@ -360,7 +357,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
                     hir_bound.trait_ref.path.res == Res::Def(DefKind::Trait, trait_ref.def_id)
                         && hir_bound.span.contains(span)
                 });
-                self.complain_about_missing_type_params(
+                self.report_missing_type_params(
                     missing_type_params,
                     trait_ref.def_id,
                     span,
diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/errors.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/errors.rs
index 3759a224ff7..45fee0fa402 100644
--- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/errors.rs
+++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/errors.rs
@@ -3,11 +3,12 @@ use rustc_data_structures::sorted_map::SortedMap;
 use rustc_data_structures::unord::UnordMap;
 use rustc_errors::codes::*;
 use rustc_errors::{
-    Applicability, Diag, ErrorGuaranteed, MultiSpan, listify, pluralize, struct_span_code_err,
+    Applicability, Diag, ErrorGuaranteed, MultiSpan, SuggestionStyle, listify, pluralize,
+    struct_span_code_err,
 };
-use rustc_hir as hir;
 use rustc_hir::def::{CtorOf, DefKind, Res};
 use rustc_hir::def_id::DefId;
+use rustc_hir::{self as hir, HirId};
 use rustc_middle::bug;
 use rustc_middle::ty::fast_reject::{TreatParams, simplify_type};
 use rustc_middle::ty::print::{PrintPolyTraitRefExt as _, PrintTraitRefExt as _};
@@ -23,6 +24,7 @@ use rustc_trait_selection::traits::{
     FulfillmentError, dyn_compatibility_violations_for_assoc_item,
 };
 use smallvec::SmallVec;
+use tracing::debug;
 
 use crate::errors::{
     self, AssocItemConstraintsNotAllowedHere, ManualImplementation, MissingTypeParams,
@@ -34,7 +36,7 @@ use crate::hir_ty_lowering::{AssocItemQSelf, HirTyLowerer};
 impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
     /// On missing type parameters, emit an E0393 error and provide a structured suggestion using
     /// the type parameter's name as a placeholder.
-    pub(crate) fn complain_about_missing_type_params(
+    pub(crate) fn report_missing_type_params(
         &self,
         missing_type_params: Vec<Symbol>,
         def_id: DefId,
@@ -56,7 +58,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
 
     /// When the code is using the `Fn` traits directly, instead of the `Fn(A) -> B` syntax, emit
     /// an error and attempt to build a reasonable structured suggestion.
-    pub(crate) fn complain_about_internal_fn_trait(
+    pub(crate) fn report_internal_fn_trait(
         &self,
         span: Span,
         trait_def_id: DefId,
@@ -112,7 +114,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         }
     }
 
-    pub(super) fn complain_about_assoc_item_not_found<I>(
+    pub(super) fn report_unresolved_assoc_item<I>(
         &self,
         all_candidates: impl Fn() -> I,
         qself: AssocItemQSelf,
@@ -132,7 +134,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
                 .filter_by_name_unhygienic(assoc_ident.name)
                 .find(|item| tcx.hygienic_eq(assoc_ident, item.ident(tcx), r.def_id()))
         }) {
-            return self.complain_about_assoc_kind_mismatch(
+            return self.report_assoc_kind_mismatch(
                 assoc_item,
                 assoc_tag,
                 assoc_ident,
@@ -331,7 +333,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         self.dcx().emit_err(err)
     }
 
-    fn complain_about_assoc_kind_mismatch(
+    fn report_assoc_kind_mismatch(
         &self,
         assoc_item: &ty::AssocItem,
         assoc_tag: ty::AssocTag,
@@ -396,12 +398,177 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         })
     }
 
-    pub(super) fn report_ambiguous_assoc(
+    pub(crate) fn report_missing_self_ty_for_resolved_path(
+        &self,
+        trait_def_id: DefId,
+        span: Span,
+        item_segment: &hir::PathSegment<'tcx>,
+        assoc_tag: ty::AssocTag,
+    ) -> ErrorGuaranteed {
+        let tcx = self.tcx();
+        let path_str = tcx.def_path_str(trait_def_id);
+
+        let def_id = self.item_def_id();
+        debug!(item_def_id = ?def_id);
+
+        // FIXME: document why/how this is different from `tcx.local_parent(def_id)`
+        let parent_def_id = tcx.hir_get_parent_item(tcx.local_def_id_to_hir_id(def_id)).to_def_id();
+        debug!(?parent_def_id);
+
+        // If the trait in segment is the same as the trait defining the item,
+        // use the `<Self as ..>` syntax in the error.
+        let is_part_of_self_trait_constraints = def_id.to_def_id() == trait_def_id;
+        let is_part_of_fn_in_self_trait = parent_def_id == trait_def_id;
+
+        let type_names = if is_part_of_self_trait_constraints || is_part_of_fn_in_self_trait {
+            vec!["Self".to_string()]
+        } else {
+            // Find all the types that have an `impl` for the trait.
+            tcx.all_impls(trait_def_id)
+                .filter_map(|impl_def_id| tcx.impl_trait_header(impl_def_id))
+                .filter(|header| {
+                    // Consider only accessible traits
+                    tcx.visibility(trait_def_id).is_accessible_from(self.item_def_id(), tcx)
+                        && header.polarity != ty::ImplPolarity::Negative
+                })
+                .map(|header| header.trait_ref.instantiate_identity().self_ty())
+                // We don't care about blanket impls.
+                .filter(|self_ty| !self_ty.has_non_region_param())
+                .map(|self_ty| tcx.erase_regions(self_ty).to_string())
+                .collect()
+        };
+        // FIXME: also look at `tcx.generics_of(self.item_def_id()).params` any that
+        // references the trait. Relevant for the first case in
+        // `src/test/ui/associated-types/associated-types-in-ambiguous-context.rs`
+        self.report_ambiguous_assoc_item_path(
+            span,
+            &type_names,
+            &[path_str],
+            item_segment.ident,
+            assoc_tag,
+        )
+    }
+
+    pub(super) fn report_unresolved_type_relative_path(
+        &self,
+        self_ty: Ty<'tcx>,
+        hir_self_ty: &hir::Ty<'_>,
+        assoc_tag: ty::AssocTag,
+        ident: Ident,
+        qpath_hir_id: HirId,
+        span: Span,
+        variant_def_id: Option<DefId>,
+    ) -> ErrorGuaranteed {
+        let tcx = self.tcx();
+        let kind_str = assoc_tag_str(assoc_tag);
+        if variant_def_id.is_some() {
+            // Variant in type position
+            let msg = format!("expected {kind_str}, found variant `{ident}`");
+            self.dcx().span_err(span, msg)
+        } else if self_ty.is_enum() {
+            let mut err = self.dcx().create_err(errors::NoVariantNamed {
+                span: ident.span,
+                ident,
+                ty: self_ty,
+            });
+
+            let adt_def = self_ty.ty_adt_def().expect("enum is not an ADT");
+            if let Some(variant_name) = find_best_match_for_name(
+                &adt_def.variants().iter().map(|variant| variant.name).collect::<Vec<Symbol>>(),
+                ident.name,
+                None,
+            ) && let Some(variant) = adt_def.variants().iter().find(|s| s.name == variant_name)
+            {
+                let mut suggestion = vec![(ident.span, variant_name.to_string())];
+                if let hir::Node::Stmt(&hir::Stmt { kind: hir::StmtKind::Semi(expr), .. })
+                | hir::Node::Expr(expr) = tcx.parent_hir_node(qpath_hir_id)
+                    && let hir::ExprKind::Struct(..) = expr.kind
+                {
+                    match variant.ctor {
+                        None => {
+                            // struct
+                            suggestion = vec![(
+                                ident.span.with_hi(expr.span.hi()),
+                                if variant.fields.is_empty() {
+                                    format!("{variant_name} {{}}")
+                                } else {
+                                    format!(
+                                        "{variant_name} {{ {} }}",
+                                        variant
+                                            .fields
+                                            .iter()
+                                            .map(|f| format!("{}: /* value */", f.name))
+                                            .collect::<Vec<_>>()
+                                            .join(", ")
+                                    )
+                                },
+                            )];
+                        }
+                        Some((hir::def::CtorKind::Fn, def_id)) => {
+                            // tuple
+                            let fn_sig = tcx.fn_sig(def_id).instantiate_identity();
+                            let inputs = fn_sig.inputs().skip_binder();
+                            suggestion = vec![(
+                                ident.span.with_hi(expr.span.hi()),
+                                format!(
+                                    "{variant_name}({})",
+                                    inputs
+                                        .iter()
+                                        .map(|i| format!("/* {i} */"))
+                                        .collect::<Vec<_>>()
+                                        .join(", ")
+                                ),
+                            )];
+                        }
+                        Some((hir::def::CtorKind::Const, _)) => {
+                            // unit
+                            suggestion = vec![(
+                                ident.span.with_hi(expr.span.hi()),
+                                variant_name.to_string(),
+                            )];
+                        }
+                    }
+                }
+                err.multipart_suggestion_verbose(
+                    "there is a variant with a similar name",
+                    suggestion,
+                    Applicability::HasPlaceholders,
+                );
+            } else {
+                err.span_label(ident.span, format!("variant not found in `{self_ty}`"));
+            }
+
+            if let Some(sp) = tcx.hir_span_if_local(adt_def.did()) {
+                err.span_label(sp, format!("variant `{ident}` not found here"));
+            }
+
+            err.emit()
+        } else if let Err(reported) = self_ty.error_reported() {
+            reported
+        } else {
+            match self.maybe_report_similar_assoc_fn(span, self_ty, hir_self_ty) {
+                Ok(()) => {}
+                Err(reported) => return reported,
+            }
+
+            let traits: Vec<_> = self.probe_traits_that_match_assoc_ty(self_ty, ident);
+
+            self.report_ambiguous_assoc_item_path(
+                span,
+                &[self_ty.to_string()],
+                &traits,
+                ident,
+                assoc_tag,
+            )
+        }
+    }
+
+    pub(super) fn report_ambiguous_assoc_item_path(
         &self,
         span: Span,
         types: &[String],
         traits: &[String],
-        name: Symbol,
+        ident: Ident,
         assoc_tag: ty::AssocTag,
     ) -> ErrorGuaranteed {
         let kind_str = assoc_tag_str(assoc_tag);
@@ -421,6 +588,8 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
                 Applicability::MachineApplicable,
             );
         } else {
+            let sugg_sp = span.until(ident.span);
+
             let mut types = types.to_vec();
             types.sort();
             let mut traits = traits.to_vec();
@@ -428,76 +597,79 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
             match (&types[..], &traits[..]) {
                 ([], []) => {
                     err.span_suggestion_verbose(
-                        span,
+                        sugg_sp,
                         format!(
                             "if there were a type named `Type` that implements a trait named \
-                             `Trait` with associated {kind_str} `{name}`, you could use the \
+                             `Trait` with associated {kind_str} `{ident}`, you could use the \
                              fully-qualified path",
                         ),
-                        format!("<Type as Trait>::{name}"),
+                        "<Type as Trait>::",
                         Applicability::HasPlaceholders,
                     );
                 }
                 ([], [trait_str]) => {
                     err.span_suggestion_verbose(
-                        span,
+                        sugg_sp,
                         format!(
                             "if there were a type named `Example` that implemented `{trait_str}`, \
                              you could use the fully-qualified path",
                         ),
-                        format!("<Example as {trait_str}>::{name}"),
+                        format!("<Example as {trait_str}>::"),
                         Applicability::HasPlaceholders,
                     );
                 }
                 ([], traits) => {
-                    err.span_suggestions(
-                        span,
+                    err.span_suggestions_with_style(
+                        sugg_sp,
                         format!(
                             "if there were a type named `Example` that implemented one of the \
-                             traits with associated {kind_str} `{name}`, you could use the \
+                             traits with associated {kind_str} `{ident}`, you could use the \
                              fully-qualified path",
                         ),
-                        traits.iter().map(|trait_str| format!("<Example as {trait_str}>::{name}")),
+                        traits.iter().map(|trait_str| format!("<Example as {trait_str}>::")),
                         Applicability::HasPlaceholders,
+                        SuggestionStyle::ShowAlways,
                     );
                 }
                 ([type_str], []) => {
                     err.span_suggestion_verbose(
-                        span,
+                        sugg_sp,
                         format!(
-                            "if there were a trait named `Example` with associated {kind_str} `{name}` \
+                            "if there were a trait named `Example` with associated {kind_str} `{ident}` \
                              implemented for `{type_str}`, you could use the fully-qualified path",
                         ),
-                        format!("<{type_str} as Example>::{name}"),
+                        format!("<{type_str} as Example>::"),
                         Applicability::HasPlaceholders,
                     );
                 }
                 (types, []) => {
-                    err.span_suggestions(
-                        span,
+                    err.span_suggestions_with_style(
+                        sugg_sp,
                         format!(
-                            "if there were a trait named `Example` with associated {kind_str} `{name}` \
+                            "if there were a trait named `Example` with associated {kind_str} `{ident}` \
                              implemented for one of the types, you could use the fully-qualified \
                              path",
                         ),
                         types
                             .into_iter()
-                            .map(|type_str| format!("<{type_str} as Example>::{name}")),
+                            .map(|type_str| format!("<{type_str} as Example>::")),
                         Applicability::HasPlaceholders,
+                        SuggestionStyle::ShowAlways,
                     );
                 }
                 (types, traits) => {
                     let mut suggestions = vec![];
                     for type_str in types {
                         for trait_str in traits {
-                            suggestions.push(format!("<{type_str} as {trait_str}>::{name}"));
+                            suggestions.push(format!("<{type_str} as {trait_str}>::"));
                         }
                     }
-                    err.span_suggestions(
-                        span,
+                    err.span_suggestions_with_style(
+                        sugg_sp,
                         "use fully-qualified syntax",
                         suggestions,
                         Applicability::MachineApplicable,
+                        SuggestionStyle::ShowAlways,
                     );
                 }
             }
@@ -505,7 +677,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         err.emit()
     }
 
-    pub(crate) fn complain_about_ambiguous_inherent_assoc(
+    pub(crate) fn report_ambiguous_inherent_assoc_item(
         &self,
         name: Ident,
         candidates: Vec<DefId>,
@@ -518,12 +690,12 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
             "multiple applicable items in scope"
         );
         err.span_label(name.span, format!("multiple `{name}` found"));
-        self.note_ambiguous_inherent_assoc_ty(&mut err, candidates, span);
+        self.note_ambiguous_inherent_assoc_item(&mut err, candidates, span);
         err.emit()
     }
 
     // FIXME(fmease): Heavily adapted from `rustc_hir_typeck::method::suggest`. Deduplicate.
-    fn note_ambiguous_inherent_assoc_ty(
+    fn note_ambiguous_inherent_assoc_item(
         &self,
         err: &mut Diag<'_>,
         candidates: Vec<DefId>,
@@ -566,7 +738,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
     }
 
     // FIXME(inherent_associated_types): Find similarly named associated types and suggest them.
-    pub(crate) fn complain_about_inherent_assoc_not_found(
+    pub(crate) fn report_unresolved_inherent_assoc_item(
         &self,
         name: Ident,
         self_ty: Ty<'tcx>,
@@ -1046,7 +1218,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         }
     }
 
-    pub fn report_prohibit_generics_error<'a>(
+    pub fn report_prohibited_generic_args<'a>(
         &self,
         segments: impl Iterator<Item = &'a hir::PathSegment<'a>> + Clone,
         args_visitors: impl Iterator<Item = &'a hir::GenericArg<'a>> + Clone,
@@ -1128,7 +1300,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         err.emit()
     }
 
-    pub fn report_trait_object_addition_traits_error(
+    pub fn report_trait_object_addition_traits(
         &self,
         regular_traits: &Vec<(ty::PolyTraitPredicate<'tcx>, SmallVec<[Span; 1]>)>,
     ) -> ErrorGuaranteed {
@@ -1171,7 +1343,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         err.emit()
     }
 
-    pub fn report_trait_object_with_no_traits_error(
+    pub fn report_trait_object_with_no_traits(
         &self,
         span: Span,
         user_written_clauses: impl IntoIterator<Item = (ty::Clause<'tcx>, Span)>,
diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/generics.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/generics.rs
index 21f0f9648ea..3a26b8331f8 100644
--- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/generics.rs
+++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/generics.rs
@@ -73,7 +73,7 @@ fn generic_arg_mismatch_err(
                     let param_name = tcx.hir_ty_param_name(param_local_id);
                     let param_type = tcx.type_of(param.def_id).instantiate_identity();
                     if param_type.is_suggestable(tcx, false) {
-                        err.span_suggestion(
+                        err.span_suggestion_verbose(
                             tcx.def_span(src_def_id),
                             "consider changing this type parameter to a const parameter",
                             format!("const {param_name}: {param_type}"),
diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/lint.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/lint.rs
index 483b61add33..9abae33ffdb 100644
--- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/lint.rs
+++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/lint.rs
@@ -103,7 +103,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
             // In case there is an associated type with the same name
             // Add the suggestion to this error
             if let Some(mut sugg) =
-                tcx.dcx().steal_non_err(self_ty.span, StashKey::AssociatedTypeSuggestion)
+                self.dcx().steal_non_err(self_ty.span, StashKey::AssociatedTypeSuggestion)
                 && let Suggestions::Enabled(ref mut s1) = diag.suggestions
                 && let Suggestions::Enabled(ref mut s2) = sugg.suggestions
             {
@@ -141,7 +141,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
 
         let generics = match tcx.hir_node_by_def_id(parent_item) {
             hir::Node::Item(hir::Item {
-                kind: hir::ItemKind::Struct(_, variant, generics),
+                kind: hir::ItemKind::Struct(_, generics, variant),
                 ..
             }) => {
                 if !variant.fields().iter().any(|field| field.hir_id == parent_hir_id) {
@@ -149,7 +149,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
                 }
                 generics
             }
-            hir::Node::Item(hir::Item { kind: hir::ItemKind::Enum(_, def, generics), .. }) => {
+            hir::Node::Item(hir::Item { kind: hir::ItemKind::Enum(_, generics, def), .. }) => {
                 if !def
                     .variants
                     .iter()
@@ -269,7 +269,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
             hir::Node::Field(field) => {
                 // Enums can't have unsized fields, fields can only have an unsized tail field.
                 if let hir::Node::Item(hir::Item {
-                    kind: hir::ItemKind::Struct(_, variant, _), ..
+                    kind: hir::ItemKind::Struct(_, _, variant), ..
                 }) = tcx.parent_hir_node(field.hir_id)
                     && variant
                         .fields()
diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
index 5e79e932015..4c65d0d0510 100644
--- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
+++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
@@ -33,27 +33,25 @@ use rustc_hir::def::{CtorKind, CtorOf, DefKind, Res};
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::{self as hir, AnonConst, GenericArg, GenericArgs, HirId};
 use rustc_infer::infer::{InferCtxt, TyCtxtInferExt};
-use rustc_infer::traits::ObligationCause;
+use rustc_infer::traits::{DynCompatibilityViolation, ObligationCause};
 use rustc_middle::middle::stability::AllowUnstable;
 use rustc_middle::mir::interpret::LitToConstInput;
 use rustc_middle::ty::print::PrintPolyTraitRefExt as _;
 use rustc_middle::ty::{
-    self, AssocTag, Const, GenericArgKind, GenericArgsRef, GenericParamDefKind, ParamEnv, Ty,
-    TyCtxt, TypeVisitableExt, TypingMode, Upcast, fold_regions,
+    self, Const, GenericArgKind, GenericArgsRef, GenericParamDefKind, ParamEnv, Ty, TyCtxt,
+    TypeVisitableExt, TypingMode, Upcast, fold_regions,
 };
 use rustc_middle::{bug, span_bug};
 use rustc_session::lint::builtin::AMBIGUOUS_ASSOCIATED_ITEMS;
 use rustc_session::parse::feature_err;
-use rustc_span::edit_distance::find_best_match_for_name;
-use rustc_span::{DUMMY_SP, Ident, Span, Symbol, kw, sym};
+use rustc_span::{DUMMY_SP, Ident, Span, kw, sym};
 use rustc_trait_selection::infer::InferCtxtExt;
 use rustc_trait_selection::traits::wf::object_region_bounds;
 use rustc_trait_selection::traits::{self, ObligationCtxt};
 use tracing::{debug, instrument};
 
-use self::errors::assoc_tag_str;
 use crate::check::check_abi_fn_ptr;
-use crate::errors::{AmbiguousLifetimeBound, BadReturnTypeNotation, NoVariantNamed};
+use crate::errors::{AmbiguousLifetimeBound, BadReturnTypeNotation};
 use crate::hir_ty_lowering::errors::{GenericsArgsErrExtend, prohibit_assoc_item_constraint};
 use crate::hir_ty_lowering::generics::{check_generic_arg_count, lower_generic_args};
 use crate::middle::resolve_bound_vars as rbv;
@@ -150,7 +148,7 @@ pub trait HirTyLowerer<'tcx> {
         assoc_ident: Ident,
     ) -> ty::EarlyBinder<'tcx, &'tcx [(ty::Clause<'tcx>, Span)]>;
 
-    /// Lower an associated type/const (from a trait) to a projection.
+    /// Lower a path to an associated item (of a trait) to a projection.
     ///
     /// This method has to be defined by the concrete lowering context because
     /// dealing with higher-ranked trait references depends on its capabilities:
@@ -162,13 +160,12 @@ pub trait HirTyLowerer<'tcx> {
     ///
     /// The canonical example of this is associated type `T::P` where `T` is a type
     /// param constrained by `T: for<'a> Trait<'a>` and where `Trait` defines `P`.
-    fn lower_assoc_shared(
+    fn lower_assoc_item_path(
         &self,
         span: Span,
         item_def_id: DefId,
         item_segment: &hir::PathSegment<'tcx>,
         poly_trait_ref: ty::PolyTraitRef<'tcx>,
-        assoc_tag: ty::AssocTag,
     ) -> Result<(DefId, GenericArgsRef<'tcx>), ErrorGuaranteed>;
 
     fn lower_fn_sig(
@@ -203,6 +200,10 @@ pub trait HirTyLowerer<'tcx> {
     {
         self
     }
+
+    /// Performs minimalistic dyn compat checks outside of bodies, but full within bodies.
+    /// Outside of bodies we could end up in cycles, so we delay most checks to later phases.
+    fn dyn_compatibility_violations(&self, trait_def_id: DefId) -> Vec<DynCompatibilityViolation>;
 }
 
 /// The "qualified self" of an associated item path.
@@ -245,39 +246,46 @@ pub enum FeedConstTy<'a, 'tcx> {
 }
 
 #[derive(Debug, Clone, Copy)]
-enum LowerAssocMode {
-    Type { permit_variants: bool },
+enum LowerTypeRelativePathMode {
+    Type(PermitVariants),
     Const,
 }
 
-impl LowerAssocMode {
+impl LowerTypeRelativePathMode {
     fn assoc_tag(self) -> ty::AssocTag {
         match self {
-            LowerAssocMode::Type { .. } => ty::AssocTag::Type,
-            LowerAssocMode::Const => ty::AssocTag::Const,
+            Self::Type(_) => ty::AssocTag::Type,
+            Self::Const => ty::AssocTag::Const,
         }
     }
 
     fn def_kind(self) -> DefKind {
         match self {
-            LowerAssocMode::Type { .. } => DefKind::AssocTy,
-            LowerAssocMode::Const => DefKind::AssocConst,
+            Self::Type(_) => DefKind::AssocTy,
+            Self::Const => DefKind::AssocConst,
         }
     }
 
-    fn permit_variants(self) -> bool {
+    fn permit_variants(self) -> PermitVariants {
         match self {
-            LowerAssocMode::Type { permit_variants } => permit_variants,
+            Self::Type(permit_variants) => permit_variants,
             // FIXME(mgca): Support paths like `Option::<T>::None` or `Option::<T>::Some` which
             // resolve to const ctors/fn items respectively.
-            LowerAssocMode::Const => false,
+            Self::Const => PermitVariants::No,
         }
     }
 }
 
+/// Whether to permit a path to resolve to an enum variant.
 #[derive(Debug, Clone, Copy)]
-enum LoweredAssoc<'tcx> {
-    Term(DefId, GenericArgsRef<'tcx>),
+pub enum PermitVariants {
+    Yes,
+    No,
+}
+
+#[derive(Debug, Clone, Copy)]
+enum TypeRelativePath<'tcx> {
+    AssocItem(DefId, GenericArgsRef<'tcx>),
     Variant { adt: Ty<'tcx>, variant_did: DefId },
 }
 
@@ -603,7 +611,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
                         if !infer_args && has_default {
                             // No type parameter provided, but a default exists.
                             if let Some(prev) =
-                                preceding_args.iter().find_map(|arg| match arg.unpack() {
+                                preceding_args.iter().find_map(|arg| match arg.kind() {
                                     GenericArgKind::Type(ty) => ty.error_reported().err(),
                                     _ => None,
                                 })
@@ -745,7 +753,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
             trait_ref.path.segments.split_last().unwrap().1.iter(),
             GenericsArgsErrExtend::None,
         );
-        self.complain_about_internal_fn_trait(span, trait_def_id, trait_segment, false);
+        self.report_internal_fn_trait(span, trait_def_id, trait_segment, false);
 
         let (generic_args, arg_count) = self.lower_generic_args_of_path(
             trait_ref.path.span,
@@ -920,7 +928,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         trait_segment: &hir::PathSegment<'tcx>,
         is_impl: bool,
     ) -> ty::TraitRef<'tcx> {
-        self.complain_about_internal_fn_trait(span, trait_def_id, trait_segment, is_impl);
+        self.report_internal_fn_trait(span, trait_def_id, trait_segment, is_impl);
 
         let (generic_args, _) =
             self.lower_generic_args_of_path(span, trait_def_id, &[], trait_segment, Some(self_ty));
@@ -933,7 +941,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
     fn probe_trait_that_defines_assoc_item(
         &self,
         trait_def_id: DefId,
-        assoc_tag: AssocTag,
+        assoc_tag: ty::AssocTag,
         assoc_ident: Ident,
     ) -> bool {
         self.tcx()
@@ -976,7 +984,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         &self,
         ty_param_def_id: LocalDefId,
         ty_param_span: Span,
-        assoc_tag: AssocTag,
+        assoc_tag: ty::AssocTag,
         assoc_ident: Ident,
         span: Span,
     ) -> Result<ty::PolyTraitRef<'tcx>, ErrorGuaranteed> {
@@ -1011,7 +1019,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         &self,
         all_candidates: impl Fn() -> I,
         qself: AssocItemQSelf,
-        assoc_tag: AssocTag,
+        assoc_tag: ty::AssocTag,
         assoc_ident: Ident,
         span: Span,
         constraint: Option<&hir::AssocItemConstraint<'tcx>>,
@@ -1026,15 +1034,14 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         });
 
         let Some(bound) = matching_candidates.next() else {
-            let reported = self.complain_about_assoc_item_not_found(
+            return Err(self.report_unresolved_assoc_item(
                 all_candidates,
                 qself,
                 assoc_tag,
                 assoc_ident,
                 span,
                 constraint,
-            );
-            return Err(reported);
+            ));
         };
         debug!(?bound);
 
@@ -1127,7 +1134,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         Ok(bound)
     }
 
-    /// Lower a [type-relative] path referring to an associated type or to an enum variant.
+    /// Lower a [type-relative](hir::QPath::TypeRelative) path in type position to a type.
     ///
     /// If the path refers to an enum variant and `permit_variants` holds,
     /// the returned type is simply the provided self type `qself_ty`.
@@ -1148,61 +1155,63 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
     /// described in the previous paragraph and their modeling of projections would likely be
     /// very similar in nature.
     ///
-    /// [type-relative]: hir::QPath::TypeRelative
     /// [#22519]: https://github.com/rust-lang/rust/issues/22519
     /// [iat]: https://github.com/rust-lang/rust/issues/8995#issuecomment-1569208403
     //
     // NOTE: When this function starts resolving `Trait::AssocTy` successfully
     // it should also start reporting the `BARE_TRAIT_OBJECTS` lint.
     #[instrument(level = "debug", skip_all, ret)]
-    pub fn lower_assoc_path_ty(
+    pub fn lower_type_relative_ty_path(
         &self,
-        hir_ref_id: HirId,
+        self_ty: Ty<'tcx>,
+        hir_self_ty: &'tcx hir::Ty<'tcx>,
+        segment: &'tcx hir::PathSegment<'tcx>,
+        qpath_hir_id: HirId,
         span: Span,
-        qself_ty: Ty<'tcx>,
-        qself: &'tcx hir::Ty<'tcx>,
-        assoc_segment: &'tcx hir::PathSegment<'tcx>,
-        permit_variants: bool,
+        permit_variants: PermitVariants,
     ) -> Result<(Ty<'tcx>, DefKind, DefId), ErrorGuaranteed> {
         let tcx = self.tcx();
-        match self.lower_assoc_path_shared(
-            hir_ref_id,
+        match self.lower_type_relative_path(
+            self_ty,
+            hir_self_ty,
+            segment,
+            qpath_hir_id,
             span,
-            qself_ty,
-            qself,
-            assoc_segment,
-            LowerAssocMode::Type { permit_variants },
+            LowerTypeRelativePathMode::Type(permit_variants),
         )? {
-            LoweredAssoc::Term(def_id, args) => {
+            TypeRelativePath::AssocItem(def_id, args) => {
                 let alias_ty = ty::AliasTy::new_from_args(tcx, def_id, args);
                 let ty = Ty::new_alias(tcx, alias_ty.kind(tcx), alias_ty);
                 Ok((ty, tcx.def_kind(def_id), def_id))
             }
-            LoweredAssoc::Variant { adt, variant_did } => Ok((adt, DefKind::Variant, variant_did)),
+            TypeRelativePath::Variant { adt, variant_did } => {
+                Ok((adt, DefKind::Variant, variant_did))
+            }
         }
     }
 
+    /// Lower a [type-relative][hir::QPath::TypeRelative] path to a (type-level) constant.
     #[instrument(level = "debug", skip_all, ret)]
-    fn lower_assoc_path_const(
+    fn lower_type_relative_const_path(
         &self,
-        hir_ref_id: HirId,
+        self_ty: Ty<'tcx>,
+        hir_self_ty: &'tcx hir::Ty<'tcx>,
+        segment: &'tcx hir::PathSegment<'tcx>,
+        qpath_hir_id: HirId,
         span: Span,
-        qself_ty: Ty<'tcx>,
-        qself: &'tcx hir::Ty<'tcx>,
-        assoc_segment: &'tcx hir::PathSegment<'tcx>,
     ) -> Result<Const<'tcx>, ErrorGuaranteed> {
         let tcx = self.tcx();
-        let (def_id, args) = match self.lower_assoc_path_shared(
-            hir_ref_id,
+        let (def_id, args) = match self.lower_type_relative_path(
+            self_ty,
+            hir_self_ty,
+            segment,
+            qpath_hir_id,
             span,
-            qself_ty,
-            qself,
-            assoc_segment,
-            LowerAssocMode::Const,
+            LowerTypeRelativePathMode::Const,
         )? {
-            LoweredAssoc::Term(def_id, args) => {
+            TypeRelativePath::AssocItem(def_id, args) => {
                 if !tcx.associated_item(def_id).is_type_const_capable(tcx) {
-                    let mut err = tcx.dcx().struct_span_err(
+                    let mut err = self.dcx().struct_span_err(
                         span,
                         "use of trait associated const without `#[type_const]`",
                     );
@@ -1213,75 +1222,134 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
             }
             // FIXME(mgca): implement support for this once ready to support all adt ctor expressions,
             // not just const ctors
-            LoweredAssoc::Variant { .. } => {
+            TypeRelativePath::Variant { .. } => {
                 span_bug!(span, "unexpected variant res for type associated const path")
             }
         };
         Ok(Const::new_unevaluated(tcx, ty::UnevaluatedConst::new(def_id, args)))
     }
 
+    /// Lower a [type-relative][hir::QPath::TypeRelative] (and type-level) path.
     #[instrument(level = "debug", skip_all, ret)]
-    fn lower_assoc_path_shared(
+    fn lower_type_relative_path(
         &self,
-        hir_ref_id: HirId,
+        self_ty: Ty<'tcx>,
+        hir_self_ty: &'tcx hir::Ty<'tcx>,
+        segment: &'tcx hir::PathSegment<'tcx>,
+        qpath_hir_id: HirId,
         span: Span,
-        qself_ty: Ty<'tcx>,
-        qself: &'tcx hir::Ty<'tcx>,
-        assoc_segment: &'tcx hir::PathSegment<'tcx>,
-        mode: LowerAssocMode,
-    ) -> Result<LoweredAssoc<'tcx>, ErrorGuaranteed> {
-        debug!(%qself_ty, ?assoc_segment.ident);
+        mode: LowerTypeRelativePathMode,
+    ) -> Result<TypeRelativePath<'tcx>, ErrorGuaranteed> {
+        debug!(%self_ty, ?segment.ident);
         let tcx = self.tcx();
 
-        let assoc_ident = assoc_segment.ident;
-
         // Check if we have an enum variant or an inherent associated type.
-        let mut variant_resolution = None;
-        if let Some(adt_def) = self.probe_adt(span, qself_ty) {
+        let mut variant_def_id = None;
+        if let Some(adt_def) = self.probe_adt(span, self_ty) {
             if adt_def.is_enum() {
                 let variant_def = adt_def
                     .variants()
                     .iter()
-                    .find(|vd| tcx.hygienic_eq(assoc_ident, vd.ident(tcx), adt_def.did()));
+                    .find(|vd| tcx.hygienic_eq(segment.ident, vd.ident(tcx), adt_def.did()));
                 if let Some(variant_def) = variant_def {
-                    if mode.permit_variants() {
-                        tcx.check_stability(variant_def.def_id, Some(hir_ref_id), span, None);
+                    if let PermitVariants::Yes = mode.permit_variants() {
+                        tcx.check_stability(variant_def.def_id, Some(qpath_hir_id), span, None);
                         let _ = self.prohibit_generic_args(
-                            slice::from_ref(assoc_segment).iter(),
-                            GenericsArgsErrExtend::EnumVariant { qself, assoc_segment, adt_def },
+                            slice::from_ref(segment).iter(),
+                            GenericsArgsErrExtend::EnumVariant {
+                                qself: hir_self_ty,
+                                assoc_segment: segment,
+                                adt_def,
+                            },
                         );
-                        return Ok(LoweredAssoc::Variant {
-                            adt: qself_ty,
+                        return Ok(TypeRelativePath::Variant {
+                            adt: self_ty,
                             variant_did: variant_def.def_id,
                         });
                     } else {
-                        variant_resolution = Some(variant_def.def_id);
+                        variant_def_id = Some(variant_def.def_id);
                     }
                 }
             }
 
             // FIXME(inherent_associated_types, #106719): Support self types other than ADTs.
-            if let Some((did, args)) = self.probe_inherent_assoc_shared(
-                assoc_segment,
+            if let Some((did, args)) = self.probe_inherent_assoc_item(
+                segment,
                 adt_def.did(),
-                qself_ty,
-                hir_ref_id,
+                self_ty,
+                qpath_hir_id,
                 span,
                 mode.assoc_tag(),
             )? {
-                return Ok(LoweredAssoc::Term(did, args));
+                return Ok(TypeRelativePath::AssocItem(did, args));
             }
         }
 
-        let qself_res = if let hir::TyKind::Path(hir::QPath::Resolved(_, path)) = &qself.kind {
-            path.res
-        } else {
-            Res::Err
+        let (item_def_id, bound) = self.resolve_type_relative_path(
+            self_ty,
+            hir_self_ty,
+            mode.assoc_tag(),
+            segment,
+            qpath_hir_id,
+            span,
+            variant_def_id,
+        )?;
+
+        let (item_def_id, args) = self.lower_assoc_item_path(span, item_def_id, segment, bound)?;
+
+        if let Some(variant_def_id) = variant_def_id {
+            tcx.node_span_lint(AMBIGUOUS_ASSOCIATED_ITEMS, qpath_hir_id, span, |lint| {
+                lint.primary_message("ambiguous associated item");
+                let mut could_refer_to = |kind: DefKind, def_id, also| {
+                    let note_msg = format!(
+                        "`{}` could{} refer to the {} defined here",
+                        segment.ident,
+                        also,
+                        tcx.def_kind_descr(kind, def_id)
+                    );
+                    lint.span_note(tcx.def_span(def_id), note_msg);
+                };
+
+                could_refer_to(DefKind::Variant, variant_def_id, "");
+                could_refer_to(mode.def_kind(), item_def_id, " also");
+
+                lint.span_suggestion(
+                    span,
+                    "use fully-qualified syntax",
+                    format!(
+                        "<{} as {}>::{}",
+                        self_ty,
+                        tcx.item_name(bound.def_id()),
+                        segment.ident
+                    ),
+                    Applicability::MachineApplicable,
+                );
+            });
+        }
+
+        Ok(TypeRelativePath::AssocItem(item_def_id, args))
+    }
+
+    /// Resolve a [type-relative](hir::QPath::TypeRelative) (and type-level) path.
+    fn resolve_type_relative_path(
+        &self,
+        self_ty: Ty<'tcx>,
+        hir_self_ty: &'tcx hir::Ty<'tcx>,
+        assoc_tag: ty::AssocTag,
+        segment: &'tcx hir::PathSegment<'tcx>,
+        qpath_hir_id: HirId,
+        span: Span,
+        variant_def_id: Option<DefId>,
+    ) -> Result<(DefId, ty::PolyTraitRef<'tcx>), ErrorGuaranteed> {
+        let tcx = self.tcx();
+
+        let self_ty_res = match hir_self_ty.kind {
+            hir::TyKind::Path(hir::QPath::Resolved(_, path)) => path.res,
+            _ => Res::Err,
         };
 
-        // Find the type of the associated item, and the trait where the associated
-        // item is declared.
-        let bound = match (qself_ty.kind(), qself_res) {
+        // Find the type of the assoc item, and the trait where the associated item is declared.
+        let bound = match (self_ty.kind(), self_ty_res) {
             (_, Res::SelfTyAlias { alias_to: impl_def_id, is_trait_impl: true, .. }) => {
                 // `Self` in an impl of a trait -- we have a concrete self type and a
                 // trait reference.
@@ -1292,14 +1360,12 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
 
                 self.probe_single_bound_for_assoc_item(
                     || {
-                        traits::supertraits(
-                            tcx,
-                            ty::Binder::dummy(trait_ref.instantiate_identity()),
-                        )
+                        let trait_ref = ty::Binder::dummy(trait_ref.instantiate_identity());
+                        traits::supertraits(tcx, trait_ref)
                     },
                     AssocItemQSelf::SelfTyAlias,
-                    mode.assoc_tag(),
-                    assoc_ident,
+                    assoc_tag,
+                    segment.ident,
                     span,
                     None,
                 )?
@@ -1309,167 +1375,33 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
                 Res::SelfTyParam { trait_: param_did } | Res::Def(DefKind::TyParam, param_did),
             ) => self.probe_single_ty_param_bound_for_assoc_item(
                 param_did.expect_local(),
-                qself.span,
-                mode.assoc_tag(),
-                assoc_ident,
+                hir_self_ty.span,
+                assoc_tag,
+                segment.ident,
                 span,
             )?,
             _ => {
-                let kind_str = assoc_tag_str(mode.assoc_tag());
-                let reported = if variant_resolution.is_some() {
-                    // Variant in type position
-                    let msg = format!("expected {kind_str}, found variant `{assoc_ident}`");
-                    self.dcx().span_err(span, msg)
-                } else if qself_ty.is_enum() {
-                    let mut err = self.dcx().create_err(NoVariantNamed {
-                        span: assoc_ident.span,
-                        ident: assoc_ident,
-                        ty: qself_ty,
-                    });
-
-                    let adt_def = qself_ty.ty_adt_def().expect("enum is not an ADT");
-                    if let Some(variant_name) = find_best_match_for_name(
-                        &adt_def
-                            .variants()
-                            .iter()
-                            .map(|variant| variant.name)
-                            .collect::<Vec<Symbol>>(),
-                        assoc_ident.name,
-                        None,
-                    ) && let Some(variant) =
-                        adt_def.variants().iter().find(|s| s.name == variant_name)
-                    {
-                        let mut suggestion = vec![(assoc_ident.span, variant_name.to_string())];
-                        if let hir::Node::Stmt(&hir::Stmt {
-                            kind: hir::StmtKind::Semi(expr), ..
-                        })
-                        | hir::Node::Expr(expr) = tcx.parent_hir_node(hir_ref_id)
-                            && let hir::ExprKind::Struct(..) = expr.kind
-                        {
-                            match variant.ctor {
-                                None => {
-                                    // struct
-                                    suggestion = vec![(
-                                        assoc_ident.span.with_hi(expr.span.hi()),
-                                        if variant.fields.is_empty() {
-                                            format!("{variant_name} {{}}")
-                                        } else {
-                                            format!(
-                                                "{variant_name} {{ {} }}",
-                                                variant
-                                                    .fields
-                                                    .iter()
-                                                    .map(|f| format!("{}: /* value */", f.name))
-                                                    .collect::<Vec<_>>()
-                                                    .join(", ")
-                                            )
-                                        },
-                                    )];
-                                }
-                                Some((hir::def::CtorKind::Fn, def_id)) => {
-                                    // tuple
-                                    let fn_sig = tcx.fn_sig(def_id).instantiate_identity();
-                                    let inputs = fn_sig.inputs().skip_binder();
-                                    suggestion = vec![(
-                                        assoc_ident.span.with_hi(expr.span.hi()),
-                                        format!(
-                                            "{variant_name}({})",
-                                            inputs
-                                                .iter()
-                                                .map(|i| format!("/* {i} */"))
-                                                .collect::<Vec<_>>()
-                                                .join(", ")
-                                        ),
-                                    )];
-                                }
-                                Some((hir::def::CtorKind::Const, _)) => {
-                                    // unit
-                                    suggestion = vec![(
-                                        assoc_ident.span.with_hi(expr.span.hi()),
-                                        variant_name.to_string(),
-                                    )];
-                                }
-                            }
-                        }
-                        err.multipart_suggestion_verbose(
-                            "there is a variant with a similar name",
-                            suggestion,
-                            Applicability::HasPlaceholders,
-                        );
-                    } else {
-                        err.span_label(
-                            assoc_ident.span,
-                            format!("variant not found in `{qself_ty}`"),
-                        );
-                    }
-
-                    if let Some(sp) = tcx.hir_span_if_local(adt_def.did()) {
-                        err.span_label(sp, format!("variant `{assoc_ident}` not found here"));
-                    }
-
-                    err.emit()
-                } else if let Err(reported) = qself_ty.error_reported() {
-                    reported
-                } else {
-                    self.maybe_report_similar_assoc_fn(span, qself_ty, qself)?;
-
-                    let traits: Vec<_> =
-                        self.probe_traits_that_match_assoc_ty(qself_ty, assoc_ident);
-
-                    // Don't print `ty::Error` to the user.
-                    self.report_ambiguous_assoc(
-                        span,
-                        &[qself_ty.to_string()],
-                        &traits,
-                        assoc_ident.name,
-                        mode.assoc_tag(),
-                    )
-                };
-                return Err(reported);
+                return Err(self.report_unresolved_type_relative_path(
+                    self_ty,
+                    hir_self_ty,
+                    assoc_tag,
+                    segment.ident,
+                    qpath_hir_id,
+                    span,
+                    variant_def_id,
+                ));
             }
         };
 
-        let trait_did = bound.def_id();
         let assoc_item = self
-            .probe_assoc_item(assoc_ident, mode.assoc_tag(), hir_ref_id, span, trait_did)
+            .probe_assoc_item(segment.ident, assoc_tag, qpath_hir_id, span, bound.def_id())
             .expect("failed to find associated item");
-        let (def_id, args) = self.lower_assoc_shared(
-            span,
-            assoc_item.def_id,
-            assoc_segment,
-            bound,
-            mode.assoc_tag(),
-        )?;
-        let result = LoweredAssoc::Term(def_id, args);
-
-        if let Some(variant_def_id) = variant_resolution {
-            tcx.node_span_lint(AMBIGUOUS_ASSOCIATED_ITEMS, hir_ref_id, span, |lint| {
-                lint.primary_message("ambiguous associated item");
-                let mut could_refer_to = |kind: DefKind, def_id, also| {
-                    let note_msg = format!(
-                        "`{}` could{} refer to the {} defined here",
-                        assoc_ident,
-                        also,
-                        tcx.def_kind_descr(kind, def_id)
-                    );
-                    lint.span_note(tcx.def_span(def_id), note_msg);
-                };
-
-                could_refer_to(DefKind::Variant, variant_def_id, "");
-                could_refer_to(mode.def_kind(), assoc_item.def_id, " also");
 
-                lint.span_suggestion(
-                    span,
-                    "use fully-qualified syntax",
-                    format!("<{} as {}>::{}", qself_ty, tcx.item_name(trait_did), assoc_ident),
-                    Applicability::MachineApplicable,
-                );
-            });
-        }
-        Ok(result)
+        Ok((assoc_item.def_id, bound))
     }
 
-    fn probe_inherent_assoc_shared(
+    /// Search for inherent associated items for use at the type level.
+    fn probe_inherent_assoc_item(
         &self,
         segment: &hir::PathSegment<'tcx>,
         adt_did: DefId,
@@ -1624,7 +1556,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
             .collect();
 
         match &applicable_candidates[..] {
-            &[] => Err(self.complain_about_inherent_assoc_not_found(
+            &[] => Err(self.report_unresolved_inherent_assoc_item(
                 name,
                 self_ty,
                 candidates,
@@ -1635,7 +1567,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
 
             &[applicable_candidate] => Ok(applicable_candidate),
 
-            &[_, ..] => Err(self.complain_about_ambiguous_inherent_assoc(
+            &[_, ..] => Err(self.report_ambiguous_inherent_assoc_item(
                 name,
                 applicable_candidates.into_iter().map(|(_, (candidate, _))| candidate).collect(),
                 span,
@@ -1763,9 +1695,9 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
             .collect()
     }
 
-    /// Lower a qualified path to a type.
+    /// Lower a [resolved][hir::QPath::Resolved] associated type path to a projection.
     #[instrument(level = "debug", skip_all)]
-    fn lower_qpath_ty(
+    fn lower_resolved_assoc_ty_path(
         &self,
         span: Span,
         opt_self_ty: Option<Ty<'tcx>>,
@@ -1773,7 +1705,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         trait_segment: Option<&hir::PathSegment<'tcx>>,
         item_segment: &hir::PathSegment<'tcx>,
     ) -> Ty<'tcx> {
-        match self.lower_qpath_shared(
+        match self.lower_resolved_assoc_item_path(
             span,
             opt_self_ty,
             item_def_id,
@@ -1788,9 +1720,9 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         }
     }
 
-    /// Lower a qualified path to a const.
+    /// Lower a [resolved][hir::QPath::Resolved] associated const path to a (type-level) constant.
     #[instrument(level = "debug", skip_all)]
-    fn lower_qpath_const(
+    fn lower_resolved_assoc_const_path(
         &self,
         span: Span,
         opt_self_ty: Option<Ty<'tcx>>,
@@ -1798,7 +1730,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         trait_segment: Option<&hir::PathSegment<'tcx>>,
         item_segment: &hir::PathSegment<'tcx>,
     ) -> Const<'tcx> {
-        match self.lower_qpath_shared(
+        match self.lower_resolved_assoc_item_path(
             span,
             opt_self_ty,
             item_def_id,
@@ -1814,8 +1746,9 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         }
     }
 
+    /// Lower a [resolved][hir::QPath::Resolved] (type-level) associated item path.
     #[instrument(level = "debug", skip_all)]
-    fn lower_qpath_shared(
+    fn lower_resolved_assoc_item_path(
         &self,
         span: Span,
         opt_self_ty: Option<Ty<'tcx>>,
@@ -1830,7 +1763,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         debug!(?trait_def_id);
 
         let Some(self_ty) = opt_self_ty else {
-            return Err(self.error_missing_qpath_self_ty(
+            return Err(self.report_missing_self_ty_for_resolved_path(
                 trait_def_id,
                 span,
                 item_segment,
@@ -1849,57 +1782,6 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         Ok((item_def_id, item_args))
     }
 
-    fn error_missing_qpath_self_ty(
-        &self,
-        trait_def_id: DefId,
-        span: Span,
-        item_segment: &hir::PathSegment<'tcx>,
-        assoc_tag: ty::AssocTag,
-    ) -> ErrorGuaranteed {
-        let tcx = self.tcx();
-        let path_str = tcx.def_path_str(trait_def_id);
-
-        let def_id = self.item_def_id();
-        debug!(item_def_id = ?def_id);
-
-        // FIXME: document why/how this is different from `tcx.local_parent(def_id)`
-        let parent_def_id = tcx.hir_get_parent_item(tcx.local_def_id_to_hir_id(def_id)).to_def_id();
-        debug!(?parent_def_id);
-
-        // If the trait in segment is the same as the trait defining the item,
-        // use the `<Self as ..>` syntax in the error.
-        let is_part_of_self_trait_constraints = def_id.to_def_id() == trait_def_id;
-        let is_part_of_fn_in_self_trait = parent_def_id == trait_def_id;
-
-        let type_names = if is_part_of_self_trait_constraints || is_part_of_fn_in_self_trait {
-            vec!["Self".to_string()]
-        } else {
-            // Find all the types that have an `impl` for the trait.
-            tcx.all_impls(trait_def_id)
-                .filter_map(|impl_def_id| tcx.impl_trait_header(impl_def_id))
-                .filter(|header| {
-                    // Consider only accessible traits
-                    tcx.visibility(trait_def_id).is_accessible_from(self.item_def_id(), tcx)
-                        && header.polarity != ty::ImplPolarity::Negative
-                })
-                .map(|header| header.trait_ref.instantiate_identity().self_ty())
-                // We don't care about blanket impls.
-                .filter(|self_ty| !self_ty.has_non_region_param())
-                .map(|self_ty| tcx.erase_regions(self_ty).to_string())
-                .collect()
-        };
-        // FIXME: also look at `tcx.generics_of(self.item_def_id()).params` any that
-        // references the trait. Relevant for the first case in
-        // `src/test/ui/associated-types/associated-types-in-ambiguous-context.rs`
-        self.report_ambiguous_assoc(
-            span,
-            &type_names,
-            &[path_str],
-            item_segment.ident.name,
-            assoc_tag,
-        )
-    }
-
     pub fn prohibit_generic_args<'a>(
         &self,
         segments: impl Iterator<Item = &'a hir::PathSegment<'a>> + Clone,
@@ -1908,7 +1790,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         let args_visitors = segments.clone().flat_map(|segment| segment.args().args);
         let mut result = Ok(());
         if let Some(_) = args_visitors.clone().next() {
-            result = Err(self.report_prohibit_generics_error(
+            result = Err(self.report_prohibited_generic_args(
                 segments.clone(),
                 args_visitors,
                 err_extend,
@@ -2069,14 +1951,14 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         generic_segments
     }
 
-    /// Lower a type `Path` to a type.
+    /// Lower a [resolved][hir::QPath::Resolved] path to a type.
     #[instrument(level = "debug", skip_all)]
-    pub fn lower_path(
+    pub fn lower_resolved_ty_path(
         &self,
         opt_self_ty: Option<Ty<'tcx>>,
         path: &hir::Path<'tcx>,
         hir_id: HirId,
-        permit_variants: bool,
+        permit_variants: PermitVariants,
     ) -> Ty<'tcx> {
         debug!(?path.res, ?opt_self_ty, ?path.segments);
         let tcx = self.tcx();
@@ -2107,7 +1989,9 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
                 );
                 self.lower_path_segment(span, did, path.segments.last().unwrap())
             }
-            Res::Def(kind @ DefKind::Variant, def_id) if permit_variants => {
+            Res::Def(kind @ DefKind::Variant, def_id)
+                if let PermitVariants::Yes = permit_variants =>
+            {
                 // Lower "variant type" as if it were a real type.
                 // The resulting `Ty` is type of the variant's enum for now.
                 assert_eq!(opt_self_ty, None);
@@ -2202,7 +2086,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
                 } else {
                     None
                 };
-                self.lower_qpath_ty(
+                self.lower_resolved_assoc_ty_path(
                     span,
                     opt_self_ty,
                     def_id,
@@ -2298,7 +2182,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         }
     }
 
-    /// Convert a [`hir::ConstArg`] to a [`ty::Const`](Const).
+    /// Lower a [`hir::ConstArg`] to a (type-level) [`ty::Const`](Const).
     #[instrument(skip(self), level = "debug")]
     pub fn lower_const_arg(
         &self,
@@ -2323,7 +2207,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
             if tcx.features().generic_const_parameter_types()
                 && (anon_const_type.has_free_regions() || anon_const_type.has_erased_regions())
             {
-                let e = tcx.dcx().span_err(
+                let e = self.dcx().span_err(
                     const_arg.span(),
                     "anonymous constants with lifetimes in their type are not yet supported",
                 );
@@ -2334,7 +2218,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
             // use this type to feed the `type_of` and query results must not contain inference
             // variables otherwise we will ICE.
             if anon_const_type.has_non_region_infer() {
-                let e = tcx.dcx().span_err(
+                let e = self.dcx().span_err(
                     const_arg.span(),
                     "anonymous constants with inferred types are not yet supported",
                 );
@@ -2344,7 +2228,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
             // We error when the type contains unsubstituted generics since we do not currently
             // give the anon const any of the generics from the parent.
             if anon_const_type.has_non_region_param() {
-                let e = tcx.dcx().span_err(
+                let e = self.dcx().span_err(
                     const_arg.span(),
                     "anonymous constants referencing generics are not yet supported",
                 );
@@ -2363,13 +2247,19 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
             hir::ConstArgKind::Path(hir::QPath::Resolved(maybe_qself, path)) => {
                 debug!(?maybe_qself, ?path);
                 let opt_self_ty = maybe_qself.as_ref().map(|qself| self.lower_ty(qself));
-                self.lower_const_path_resolved(opt_self_ty, path, hir_id)
+                self.lower_resolved_const_path(opt_self_ty, path, hir_id)
             }
-            hir::ConstArgKind::Path(hir::QPath::TypeRelative(qself, segment)) => {
-                debug!(?qself, ?segment);
-                let ty = self.lower_ty(qself);
-                self.lower_assoc_path_const(hir_id, const_arg.span(), ty, qself, segment)
-                    .unwrap_or_else(|guar| Const::new_error(tcx, guar))
+            hir::ConstArgKind::Path(hir::QPath::TypeRelative(hir_self_ty, segment)) => {
+                debug!(?hir_self_ty, ?segment);
+                let self_ty = self.lower_ty(hir_self_ty);
+                self.lower_type_relative_const_path(
+                    self_ty,
+                    hir_self_ty,
+                    segment,
+                    hir_id,
+                    const_arg.span(),
+                )
+                .unwrap_or_else(|guar| Const::new_error(tcx, guar))
             }
             hir::ConstArgKind::Path(qpath @ hir::QPath::LangItem(..)) => {
                 ty::Const::new_error_with_message(
@@ -2383,7 +2273,8 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         }
     }
 
-    fn lower_const_path_resolved(
+    /// Lower a [resolved][hir::QPath::Resolved] path to a (type-level) constant.
+    fn lower_resolved_const_path(
         &self,
         opt_self_ty: Option<Ty<'tcx>>,
         path: &hir::Path<'tcx>,
@@ -2420,7 +2311,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
                 } else {
                     None
                 };
-                self.lower_qpath_const(
+                self.lower_resolved_assoc_const_path(
                     span,
                     opt_self_ty,
                     did,
@@ -2630,7 +2521,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
             hir::TyKind::Path(hir::QPath::Resolved(maybe_qself, path)) => {
                 debug!(?maybe_qself, ?path);
                 let opt_self_ty = maybe_qself.as_ref().map(|qself| self.lower_ty(qself));
-                self.lower_path(opt_self_ty, path, hir_ty.hir_id, false)
+                self.lower_resolved_ty_path(opt_self_ty, path, hir_ty.hir_id, PermitVariants::No)
             }
             &hir::TyKind::OpaqueDef(opaque_ty) => {
                 // If this is an RPITIT and we are using the new RPITIT lowering scheme, we
@@ -2684,15 +2575,22 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
                 let guar = self.dcx().emit_err(BadReturnTypeNotation { span: hir_ty.span });
                 Ty::new_error(tcx, guar)
             }
-            hir::TyKind::Path(hir::QPath::TypeRelative(qself, segment)) => {
-                debug!(?qself, ?segment);
-                let ty = self.lower_ty(qself);
-                self.lower_assoc_path_ty(hir_ty.hir_id, hir_ty.span, ty, qself, segment, false)
-                    .map(|(ty, _, _)| ty)
-                    .unwrap_or_else(|guar| Ty::new_error(tcx, guar))
+            hir::TyKind::Path(hir::QPath::TypeRelative(hir_self_ty, segment)) => {
+                debug!(?hir_self_ty, ?segment);
+                let self_ty = self.lower_ty(hir_self_ty);
+                self.lower_type_relative_ty_path(
+                    self_ty,
+                    hir_self_ty,
+                    segment,
+                    hir_ty.hir_id,
+                    hir_ty.span,
+                    PermitVariants::No,
+                )
+                .map(|(ty, _, _)| ty)
+                .unwrap_or_else(|guar| Ty::new_error(tcx, guar))
             }
             &hir::TyKind::Path(hir::QPath::LangItem(lang_item, span)) => {
-                let def_id = tcx.require_lang_item(lang_item, Some(span));
+                let def_id = tcx.require_lang_item(lang_item, span);
                 let (args, _) = self.lower_generic_args_of_path(
                     span,
                     def_id,
diff --git a/compiler/rustc_hir_analysis/src/hir_wf_check.rs b/compiler/rustc_hir_analysis/src/hir_wf_check.rs
index 64c1a78bd1c..4633f3951a7 100644
--- a/compiler/rustc_hir_analysis/src/hir_wf_check.rs
+++ b/compiler/rustc_hir_analysis/src/hir_wf_check.rs
@@ -145,9 +145,9 @@ fn diagnostic_hir_wf_check<'tcx>(
                 ref item => bug!("Unexpected TraitItem {:?}", item),
             },
             hir::Node::Item(item) => match item.kind {
-                hir::ItemKind::TyAlias(_, ty, _)
-                | hir::ItemKind::Static(_, ty, _, _)
-                | hir::ItemKind::Const(_, ty, _, _) => vec![ty],
+                hir::ItemKind::TyAlias(_, _, ty)
+                | hir::ItemKind::Static(_, _, ty, _)
+                | hir::ItemKind::Const(_, _, ty, _) => vec![ty],
                 hir::ItemKind::Impl(impl_) => match &impl_.of_trait {
                     Some(t) => t
                         .path
diff --git a/compiler/rustc_hir_analysis/src/lib.rs b/compiler/rustc_hir_analysis/src/lib.rs
index 010c6c376fe..a92ee89186c 100644
--- a/compiler/rustc_hir_analysis/src/lib.rs
+++ b/compiler/rustc_hir_analysis/src/lib.rs
@@ -182,9 +182,7 @@ pub fn check_crate(tcx: TyCtxt<'_>) {
         // what we are intending to discard, to help future type-based refactoring.
         type R = Result<(), ErrorGuaranteed>;
 
-        tcx.par_hir_for_each_module(|module| {
-            let _: R = tcx.ensure_ok().check_mod_type_wf(module);
-        });
+        let _: R = tcx.ensure_ok().check_type_wf(());
 
         for &trait_def_id in tcx.all_local_trait_impls(()).keys() {
             let _: R = tcx.ensure_ok().coherent_trait(trait_def_id);
@@ -194,16 +192,18 @@ pub fn check_crate(tcx: TyCtxt<'_>) {
         let _: R = tcx.ensure_ok().crate_inherent_impls_overlap_check(());
     });
 
-    // Make sure we evaluate all static and (non-associated) const items, even if unused.
-    // If any of these fail to evaluate, we do not want this crate to pass compilation.
     tcx.par_hir_body_owners(|item_def_id| {
         let def_kind = tcx.def_kind(item_def_id);
+        // Make sure we evaluate all static and (non-associated) const items, even if unused.
+        // If any of these fail to evaluate, we do not want this crate to pass compilation.
         match def_kind {
             DefKind::Static { .. } => {
                 tcx.ensure_ok().eval_static_initializer(item_def_id);
                 check::maybe_check_static_with_link_section(tcx, item_def_id);
             }
-            DefKind::Const if tcx.generics_of(item_def_id).is_empty() => {
+            DefKind::Const if !tcx.generics_of(item_def_id).own_requires_monomorphization() => {
+                // FIXME(generic_const_items): Passing empty instead of identity args is fishy but
+                //                             seems to be fine for now. Revisit this!
                 let instance = ty::Instance::new_raw(item_def_id.into(), ty::GenericArgs::empty());
                 let cid = GlobalId { instance, promoted: None };
                 let typing_env = ty::TypingEnv::fully_monomorphized();
@@ -215,6 +215,11 @@ pub fn check_crate(tcx: TyCtxt<'_>) {
         if !matches!(def_kind, DefKind::AnonConst) {
             tcx.ensure_ok().typeck(item_def_id);
         }
+        // Ensure we generate the new `DefId` before finishing `check_crate`.
+        // Afterwards we freeze the list of `DefId`s.
+        if tcx.needs_coroutine_by_move_body_def_id(item_def_id.to_def_id()) {
+            tcx.ensure_done().coroutine_by_move_body_def_id(item_def_id);
+        }
     });
 
     if tcx.features().rustc_attrs() {
diff --git a/compiler/rustc_hir_analysis/src/outlives/implicit_infer.rs b/compiler/rustc_hir_analysis/src/outlives/implicit_infer.rs
index c99eb12efcc..4f35b87be30 100644
--- a/compiler/rustc_hir_analysis/src/outlives/implicit_infer.rs
+++ b/compiler/rustc_hir_analysis/src/outlives/implicit_infer.rs
@@ -119,7 +119,7 @@ fn insert_required_predicates_to_be_wf<'tcx>(
     explicit_map: &mut ExplicitPredicatesMap<'tcx>,
 ) {
     for arg in ty.walk() {
-        let leaf_ty = match arg.unpack() {
+        let leaf_ty = match arg.kind() {
             GenericArgKind::Type(ty) => ty,
 
             // No predicates from lifetimes or constants, except potentially
@@ -299,7 +299,7 @@ fn check_explicit_predicates<'tcx>(
         // binding) and thus infer an outlives requirement that `X:
         // 'b`.
         if let Some(self_ty) = ignored_self_ty
-            && let GenericArgKind::Type(ty) = outlives_predicate.0.unpack()
+            && let GenericArgKind::Type(ty) = outlives_predicate.0.kind()
             && ty.walk().any(|arg| arg == self_ty.into())
         {
             debug!("skipping self ty = {ty:?}");
diff --git a/compiler/rustc_hir_analysis/src/outlives/mod.rs b/compiler/rustc_hir_analysis/src/outlives/mod.rs
index daa908c8c78..499f5572f47 100644
--- a/compiler/rustc_hir_analysis/src/outlives/mod.rs
+++ b/compiler/rustc_hir_analysis/src/outlives/mod.rs
@@ -69,8 +69,8 @@ fn inferred_outlives_crate(tcx: TyCtxt<'_>, (): ()) -> CratePredicatesMap<'_> {
         .map(|(&def_id, set)| {
             let predicates =
                 &*tcx.arena.alloc_from_iter(set.as_ref().skip_binder().iter().filter_map(
-                    |(ty::OutlivesPredicate(kind1, region2), &span)| {
-                        match kind1.unpack() {
+                    |(ty::OutlivesPredicate(arg1, region2), &span)| {
+                        match arg1.kind() {
                             GenericArgKind::Type(ty1) => Some((
                                 ty::ClauseKind::TypeOutlives(ty::OutlivesPredicate(ty1, *region2))
                                     .upcast(tcx),
diff --git a/compiler/rustc_hir_analysis/src/outlives/utils.rs b/compiler/rustc_hir_analysis/src/outlives/utils.rs
index 044fb64ca82..301f5de9424 100644
--- a/compiler/rustc_hir_analysis/src/outlives/utils.rs
+++ b/compiler/rustc_hir_analysis/src/outlives/utils.rs
@@ -14,7 +14,7 @@ pub(crate) type RequiredPredicates<'tcx> =
 /// outlives_component and add it to `required_predicates`
 pub(crate) fn insert_outlives_predicate<'tcx>(
     tcx: TyCtxt<'tcx>,
-    kind: GenericArg<'tcx>,
+    arg: GenericArg<'tcx>,
     outlived_region: Region<'tcx>,
     span: Span,
     required_predicates: &mut RequiredPredicates<'tcx>,
@@ -25,7 +25,7 @@ pub(crate) fn insert_outlives_predicate<'tcx>(
         return;
     }
 
-    match kind.unpack() {
+    match arg.kind() {
         GenericArgKind::Type(ty) => {
             // `T: 'outlived_region` for some type `T`
             // But T could be a lot of things:
@@ -135,7 +135,7 @@ pub(crate) fn insert_outlives_predicate<'tcx>(
             if !is_free_region(r) {
                 return;
             }
-            required_predicates.entry(ty::OutlivesPredicate(kind, outlived_region)).or_insert(span);
+            required_predicates.entry(ty::OutlivesPredicate(arg, outlived_region)).or_insert(span);
         }
 
         GenericArgKind::Const(_) => {
diff --git a/compiler/rustc_hir_analysis/src/variance/constraints.rs b/compiler/rustc_hir_analysis/src/variance/constraints.rs
index 92cfece77c4..960ec7f66ab 100644
--- a/compiler/rustc_hir_analysis/src/variance/constraints.rs
+++ b/compiler/rustc_hir_analysis/src/variance/constraints.rs
@@ -200,8 +200,8 @@ impl<'a, 'tcx> ConstraintContext<'a, 'tcx> {
         // Trait are always invariant so we can take advantage of that.
         let variance_i = self.invariant(variance);
 
-        for k in args {
-            match k.unpack() {
+        for arg in args {
+            match arg.kind() {
                 GenericArgKind::Lifetime(lt) => {
                     self.add_constraints_from_region(current, lt, variance_i)
                 }
@@ -294,7 +294,7 @@ impl<'a, 'tcx> ConstraintContext<'a, 'tcx> {
                 }
 
                 for projection in data.projection_bounds() {
-                    match projection.skip_binder().term.unpack() {
+                    match projection.skip_binder().term.kind() {
                         ty::TermKind::Ty(ty) => {
                             self.add_constraints_from_ty(current, ty, self.invariant);
                         }
@@ -373,7 +373,7 @@ impl<'a, 'tcx> ConstraintContext<'a, 'tcx> {
             (None, Some(self.tcx().variances_of(def_id)))
         };
 
-        for (i, k) in args.iter().enumerate() {
+        for (i, arg) in args.iter().enumerate() {
             let variance_decl = if let Some(InferredIndex(start)) = local {
                 // Parameter on an item defined within current crate:
                 // variance not yet inferred, so return a symbolic
@@ -389,7 +389,7 @@ impl<'a, 'tcx> ConstraintContext<'a, 'tcx> {
                 "add_constraints_from_args: variance_decl={:?} variance_i={:?}",
                 variance_decl, variance_i
             );
-            match k.unpack() {
+            match arg.kind() {
                 GenericArgKind::Lifetime(lt) => {
                     self.add_constraints_from_region(current, lt, variance_i)
                 }
diff --git a/compiler/rustc_hir_pretty/src/lib.rs b/compiler/rustc_hir_pretty/src/lib.rs
index 04f9c831b0a..b23b3125c59 100644
--- a/compiler/rustc_hir_pretty/src/lib.rs
+++ b/compiler/rustc_hir_pretty/src/lib.rs
@@ -477,10 +477,10 @@ impl<'a> State<'a> {
             hir::ForeignItemKind::Fn(sig, arg_idents, generics) => {
                 let (cb, ib) = self.head("");
                 self.print_fn(
-                    sig.decl,
                     sig.header,
                     Some(item.ident.name),
                     generics,
+                    sig.decl,
                     arg_idents,
                     None,
                 );
@@ -593,7 +593,7 @@ impl<'a> State<'a> {
                 self.end(ib);
                 self.end(cb);
             }
-            hir::ItemKind::Static(ident, ty, m, expr) => {
+            hir::ItemKind::Static(m, ident, ty, expr) => {
                 let (cb, ib) = self.head("static");
                 if m.is_mut() {
                     self.word_space("mut");
@@ -609,7 +609,7 @@ impl<'a> State<'a> {
                 self.word(";");
                 self.end(cb);
             }
-            hir::ItemKind::Const(ident, ty, generics, expr) => {
+            hir::ItemKind::Const(ident, generics, ty, expr) => {
                 let (cb, ib) = self.head("const");
                 self.print_ident(ident);
                 self.print_generic_params(generics.params);
@@ -626,7 +626,7 @@ impl<'a> State<'a> {
             }
             hir::ItemKind::Fn { ident, sig, generics, body, .. } => {
                 let (cb, ib) = self.head("");
-                self.print_fn(sig.decl, sig.header, Some(ident.name), generics, &[], Some(body));
+                self.print_fn(sig.header, Some(ident.name), generics, sig.decl, &[], Some(body));
                 self.word(" ");
                 self.end(ib);
                 self.end(cb);
@@ -660,7 +660,7 @@ impl<'a> State<'a> {
                 self.end(cb);
                 self.end(ib);
             }
-            hir::ItemKind::TyAlias(ident, ty, generics) => {
+            hir::ItemKind::TyAlias(ident, generics, ty) => {
                 let (cb, ib) = self.head("type");
                 self.print_ident(ident);
                 self.print_generic_params(generics.params);
@@ -673,16 +673,16 @@ impl<'a> State<'a> {
                 self.word(";");
                 self.end(cb);
             }
-            hir::ItemKind::Enum(ident, ref enum_definition, params) => {
-                self.print_enum_def(enum_definition, params, ident.name, item.span);
+            hir::ItemKind::Enum(ident, generics, ref enum_def) => {
+                self.print_enum_def(ident.name, generics, enum_def, item.span);
             }
-            hir::ItemKind::Struct(ident, ref struct_def, generics) => {
+            hir::ItemKind::Struct(ident, generics, ref struct_def) => {
                 let (cb, ib) = self.head("struct");
-                self.print_struct(struct_def, generics, ident.name, item.span, true, cb, ib);
+                self.print_struct(ident.name, generics, struct_def, item.span, true, cb, ib);
             }
-            hir::ItemKind::Union(ident, ref struct_def, generics) => {
+            hir::ItemKind::Union(ident, generics, ref struct_def) => {
                 let (cb, ib) = self.head("union");
-                self.print_struct(struct_def, generics, ident.name, item.span, true, cb, ib);
+                self.print_struct(ident.name, generics, struct_def, item.span, true, cb, ib);
             }
             hir::ItemKind::Impl(&hir::Impl {
                 constness,
@@ -791,9 +791,9 @@ impl<'a> State<'a> {
 
     fn print_enum_def(
         &mut self,
-        enum_definition: &hir::EnumDef<'_>,
-        generics: &hir::Generics<'_>,
         name: Symbol,
+        generics: &hir::Generics<'_>,
+        enum_def: &hir::EnumDef<'_>,
         span: rustc_span::Span,
     ) {
         let (cb, ib) = self.head("enum");
@@ -801,7 +801,7 @@ impl<'a> State<'a> {
         self.print_generic_params(generics.params);
         self.print_where_clause(generics);
         self.space();
-        self.print_variants(enum_definition.variants, span, cb, ib);
+        self.print_variants(enum_def.variants, span, cb, ib);
     }
 
     fn print_variants(
@@ -834,9 +834,9 @@ impl<'a> State<'a> {
 
     fn print_struct(
         &mut self,
-        struct_def: &hir::VariantData<'_>,
-        generics: &hir::Generics<'_>,
         name: Symbol,
+        generics: &hir::Generics<'_>,
+        struct_def: &hir::VariantData<'_>,
         span: rustc_span::Span,
         print_finalizer: bool,
         cb: BoxMarker,
@@ -886,7 +886,7 @@ impl<'a> State<'a> {
     pub fn print_variant(&mut self, v: &hir::Variant<'_>) {
         let (cb, ib) = self.head("");
         let generics = hir::Generics::empty();
-        self.print_struct(&v.data, generics, v.ident.name, v.span, false, cb, ib);
+        self.print_struct(v.ident.name, generics, &v.data, v.span, false, cb, ib);
         if let Some(ref d) = v.disr_expr {
             self.space();
             self.word_space("=");
@@ -902,7 +902,7 @@ impl<'a> State<'a> {
         arg_idents: &[Option<Ident>],
         body_id: Option<hir::BodyId>,
     ) {
-        self.print_fn(m.decl, m.header, Some(ident.name), generics, arg_idents, body_id);
+        self.print_fn(m.header, Some(ident.name), generics, m.decl, arg_idents, body_id);
     }
 
     fn print_trait_item(&mut self, ti: &hir::TraitItem<'_>) {
@@ -2141,10 +2141,10 @@ impl<'a> State<'a> {
 
     fn print_fn(
         &mut self,
-        decl: &hir::FnDecl<'_>,
         header: hir::FnHeader,
         name: Option<Symbol>,
         generics: &hir::Generics<'_>,
+        decl: &hir::FnDecl<'_>,
         arg_idents: &[Option<Ident>],
         body_id: Option<hir::BodyId>,
     ) {
@@ -2483,7 +2483,6 @@ impl<'a> State<'a> {
         self.print_formal_generic_params(generic_params);
         let generics = hir::Generics::empty();
         self.print_fn(
-            decl,
             hir::FnHeader {
                 safety: safety.into(),
                 abi,
@@ -2492,6 +2491,7 @@ impl<'a> State<'a> {
             },
             name,
             generics,
+            decl,
             arg_idents,
             None,
         );
diff --git a/compiler/rustc_hir_typeck/Cargo.toml b/compiler/rustc_hir_typeck/Cargo.toml
index 40fb2d6a106..c963f0ddefd 100644
--- a/compiler/rustc_hir_typeck/Cargo.toml
+++ b/compiler/rustc_hir_typeck/Cargo.toml
@@ -9,6 +9,7 @@ itertools = "0.12"
 rustc_abi = { path = "../rustc_abi" }
 rustc_ast = { path = "../rustc_ast" }
 rustc_attr_data_structures = { path = "../rustc_attr_data_structures" }
+rustc_attr_parsing = { path = "../rustc_attr_parsing" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
diff --git a/compiler/rustc_hir_typeck/messages.ftl b/compiler/rustc_hir_typeck/messages.ftl
index 23309102c4d..6c33dfb4ec0 100644
--- a/compiler/rustc_hir_typeck/messages.ftl
+++ b/compiler/rustc_hir_typeck/messages.ftl
@@ -137,6 +137,18 @@ hir_typeck_lossy_provenance_ptr2int =
 
 hir_typeck_missing_parentheses_in_range = can't call method `{$method_name}` on type `{$ty_str}`
 
+hir_typeck_naked_asm_outside_naked_fn =
+    the `naked_asm!` macro can only be used in functions marked with `#[unsafe(naked)]`
+
+hir_typeck_naked_functions_asm_block =
+    naked functions must contain a single `naked_asm!` invocation
+    .label_multiple_asm = multiple `naked_asm!` invocations are not allowed in naked functions
+    .label_non_asm = not allowed in naked functions
+
+hir_typeck_naked_functions_must_naked_asm =
+    the `asm!` macro is not allowed in naked functions
+    .label = consider using the `naked_asm!` macro instead
+
 hir_typeck_never_type_fallback_flowing_into_unsafe_call = never type fallback affects this call to an `unsafe` function
     .help = specify the type explicitly
 hir_typeck_never_type_fallback_flowing_into_unsafe_deref = never type fallback affects this raw pointer dereference
@@ -159,6 +171,9 @@ hir_typeck_no_field_on_variant = no field named `{$field}` on enum variant `{$co
 hir_typeck_no_field_on_variant_enum = this enum variant...
 hir_typeck_no_field_on_variant_field = ...does not have this field
 
+hir_typeck_no_patterns =
+    patterns not allowed in naked function parameters
+
 hir_typeck_note_caller_chooses_ty_for_ty_param = the caller chooses a type for `{$ty_param_name}` which can be different from `{$found_ty}`
 
 hir_typeck_note_edition_guide = for more on editions, read https://doc.rust-lang.org/edition-guide
@@ -167,9 +182,13 @@ hir_typeck_option_result_asref = use `{$def_path}::as_ref` to convert `{$expecte
 hir_typeck_option_result_cloned = use `{$def_path}::cloned` to clone the value inside the `{$def_path}`
 hir_typeck_option_result_copied = use `{$def_path}::copied` to copy the value inside the `{$def_path}`
 
+hir_typeck_params_not_allowed =
+    referencing function parameters is not allowed in naked functions
+    .help = follow the calling convention in asm block to use parameters
+
 hir_typeck_pass_to_variadic_function = can't pass `{$ty}` to variadic function
     .suggestion = cast the value to `{$cast_ty}`
-    .teach_help = certain types, like `{$ty}`, must be casted before passing them to a variadic function, because of arcane ABI rules dictated by the C standard
+    .teach_help = certain types, like `{$ty}`, must be cast before passing them to a variadic function to match the implicit cast that a C compiler would perform as part of C's numeric promotion rules
 
 hir_typeck_ptr_cast_add_auto_to_object = cannot add {$traits_len ->
     [1] auto trait {$traits}
diff --git a/compiler/rustc_hir_typeck/src/callee.rs b/compiler/rustc_hir_typeck/src/callee.rs
index f555d116c52..d173fe7c2c2 100644
--- a/compiler/rustc_hir_typeck/src/callee.rs
+++ b/compiler/rustc_hir_typeck/src/callee.rs
@@ -508,7 +508,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             if let Some(ty) = fn_sig.inputs().last().copied() {
                 self.register_bound(
                     ty,
-                    self.tcx.require_lang_item(hir::LangItem::Tuple, Some(sp)),
+                    self.tcx.require_lang_item(hir::LangItem::Tuple, sp),
                     self.cause(sp, ObligationCauseCode::RustCall),
                 );
                 self.require_type_is_sized(ty, sp, ObligationCauseCode::RustCall);
diff --git a/compiler/rustc_hir_typeck/src/cast.rs b/compiler/rustc_hir_typeck/src/cast.rs
index c044c4f7c37..e17cfc15a43 100644
--- a/compiler/rustc_hir_typeck/src/cast.rs
+++ b/compiler/rustc_hir_typeck/src/cast.rs
@@ -380,20 +380,21 @@ impl<'a, 'tcx> CastCheck<'tcx> {
                 err.span_label(self.span, "invalid cast");
                 if self.expr_ty.is_numeric() {
                     if self.expr_ty == fcx.tcx.types.u32 {
-                        match fcx.tcx.sess.source_map().span_to_snippet(self.expr.span) {
-                            Ok(snippet) => err.span_suggestion(
-                                self.span,
-                                "try `char::from_u32` instead",
-                                format!("char::from_u32({snippet})"),
-                                Applicability::MachineApplicable,
-                            ),
-
-                            Err(_) => err.span_help(self.span, "try `char::from_u32` instead"),
-                        };
+                        err.multipart_suggestion(
+                            "consider using `char::from_u32` instead",
+                            vec![
+                                (self.expr_span.shrink_to_lo(), "char::from_u32(".to_string()),
+                                (self.expr_span.shrink_to_hi().to(self.cast_span), ")".to_string()),
+                            ],
+                            Applicability::MachineApplicable,
+                        );
                     } else if self.expr_ty == fcx.tcx.types.i8 {
-                        err.span_help(self.span, "try casting from `u8` instead");
+                        err.span_help(self.span, "consider casting from `u8` instead");
                     } else {
-                        err.span_help(self.span, "try `char::from_u32` instead (via a `u32`)");
+                        err.span_help(
+                            self.span,
+                            "consider using `char::from_u32` instead (via a `u32`)",
+                        );
                     };
                 }
                 err.emit();
@@ -408,6 +409,16 @@ impl<'a, 'tcx> CastCheck<'tcx> {
                     self.expr_ty,
                     fcx.ty_to_string(self.cast_ty)
                 );
+
+                if let Ok(snippet) = fcx.tcx.sess.source_map().span_to_snippet(self.expr_span)
+                    && matches!(self.expr.kind, ExprKind::AddrOf(..))
+                {
+                    err.note(format!(
+                        "casting reference expression `{}` because `&` binds tighter than `as`",
+                        snippet
+                    ));
+                }
+
                 let mut sugg = None;
                 let mut sugg_mutref = false;
                 if let ty::Ref(reg, cast_ty, mutbl) = *self.cast_ty.kind() {
@@ -484,11 +495,8 @@ impl<'a, 'tcx> CastCheck<'tcx> {
                     self.cast_ty.kind(),
                     ty::FnDef(..) | ty::FnPtr(..) | ty::Closure(..)
                 ) {
-                    let mut label = true;
                     // Check `impl From<self.expr_ty> for self.cast_ty {}` for accurate suggestion:
-                    if let Ok(snippet) = fcx.tcx.sess.source_map().span_to_snippet(self.expr_span)
-                        && let Some(from_trait) = fcx.tcx.get_diagnostic_item(sym::From)
-                    {
+                    if let Some(from_trait) = fcx.tcx.get_diagnostic_item(sym::From) {
                         let ty = fcx.resolve_vars_if_possible(self.cast_ty);
                         let expr_ty = fcx.resolve_vars_if_possible(self.expr_ty);
                         if fcx
@@ -496,26 +504,22 @@ impl<'a, 'tcx> CastCheck<'tcx> {
                             .type_implements_trait(from_trait, [ty, expr_ty], fcx.param_env)
                             .must_apply_modulo_regions()
                         {
-                            label = false;
-                            if let ty::Adt(def, args) = self.cast_ty.kind() {
-                                err.span_suggestion_verbose(
-                                    self.span,
-                                    "consider using the `From` trait instead",
-                                    format!(
-                                        "{}::from({})",
-                                        fcx.tcx.value_path_str_with_args(def.did(), args),
-                                        snippet
-                                    ),
-                                    Applicability::MaybeIncorrect,
-                                );
+                            let to_ty = if let ty::Adt(def, args) = self.cast_ty.kind() {
+                                fcx.tcx.value_path_str_with_args(def.did(), args)
                             } else {
-                                err.span_suggestion(
-                                    self.span,
-                                    "consider using the `From` trait instead",
-                                    format!("{}::from({})", self.cast_ty, snippet),
-                                    Applicability::MaybeIncorrect,
-                                );
+                                self.cast_ty.to_string()
                             };
+                            err.multipart_suggestion(
+                                "consider using the `From` trait instead",
+                                vec![
+                                    (self.expr_span.shrink_to_lo(), format!("{to_ty}::from(")),
+                                    (
+                                        self.expr_span.shrink_to_hi().to(self.cast_span),
+                                        ")".to_string(),
+                                    ),
+                                ],
+                                Applicability::MaybeIncorrect,
+                            );
                         }
                     }
 
@@ -538,11 +542,7 @@ impl<'a, 'tcx> CastCheck<'tcx> {
                         )
                     };
 
-                    if label {
-                        err.span_label(self.span, msg);
-                    } else {
-                        err.note(msg);
-                    }
+                    err.span_label(self.span, msg);
 
                     if let Some(note) = note {
                         err.note(note);
@@ -644,38 +644,22 @@ impl<'a, 'tcx> CastCheck<'tcx> {
         match self.expr_ty.kind() {
             ty::Ref(_, _, mt) => {
                 let mtstr = mt.prefix_str();
-                match fcx.tcx.sess.source_map().span_to_snippet(self.cast_span) {
-                    Ok(s) => {
-                        err.span_suggestion(
-                            self.cast_span,
-                            "try casting to a reference instead",
-                            format!("&{mtstr}{s}"),
-                            Applicability::MachineApplicable,
-                        );
-                    }
-                    Err(_) => {
-                        let msg = format!("did you mean `&{mtstr}{tstr}`?");
-                        err.span_help(self.cast_span, msg);
-                    }
-                }
+                err.span_suggestion_verbose(
+                    self.cast_span.shrink_to_lo(),
+                    "consider casting to a reference instead",
+                    format!("&{mtstr}"),
+                    Applicability::MachineApplicable,
+                );
             }
             ty::Adt(def, ..) if def.is_box() => {
-                match fcx.tcx.sess.source_map().span_to_snippet(self.cast_span) {
-                    Ok(s) => {
-                        err.span_suggestion(
-                            self.cast_span,
-                            "you can cast to a `Box` instead",
-                            format!("Box<{s}>"),
-                            Applicability::MachineApplicable,
-                        );
-                    }
-                    Err(_) => {
-                        err.span_help(
-                            self.cast_span,
-                            format!("you might have meant `Box<{tstr}>`"),
-                        );
-                    }
-                }
+                err.multipart_suggestion(
+                    "you can cast to a `Box` instead",
+                    vec![
+                        (self.cast_span.shrink_to_lo(), "Box<".to_string()),
+                        (self.cast_span.shrink_to_hi(), ">".to_string()),
+                    ],
+                    Applicability::MachineApplicable,
+                );
             }
             _ => {
                 err.span_help(self.expr_span, "consider using a box or reference as appropriate");
diff --git a/compiler/rustc_hir_typeck/src/check.rs b/compiler/rustc_hir_typeck/src/check.rs
index 99103f14d68..ac42eebf08c 100644
--- a/compiler/rustc_hir_typeck/src/check.rs
+++ b/compiler/rustc_hir_typeck/src/check.rs
@@ -57,7 +57,7 @@ pub(super) fn check_fn<'a, 'tcx>(
     // (as it's created inside the body itself, not passed in from outside).
     let maybe_va_list = fn_sig.c_variadic.then(|| {
         let span = body.params.last().unwrap().span;
-        let va_list_did = tcx.require_lang_item(LangItem::VaList, Some(span));
+        let va_list_did = tcx.require_lang_item(LangItem::VaList, span);
         let region = fcx.next_region_var(RegionVariableOrigin::MiscVariable(span));
 
         tcx.type_of(va_list_did).instantiate(tcx, &[region.into()])
@@ -178,7 +178,7 @@ fn check_panic_info_fn(tcx: TyCtxt<'_>, fn_id: LocalDefId, fn_sig: ty::FnSig<'_>
         tcx.dcx().span_err(span, "should have no const parameters");
     }
 
-    let panic_info_did = tcx.require_lang_item(hir::LangItem::PanicInfo, Some(span));
+    let panic_info_did = tcx.require_lang_item(hir::LangItem::PanicInfo, span);
 
     // build type `for<'a, 'b> fn(&'a PanicInfo<'b>) -> !`
     let panic_info_ty = tcx.type_of(panic_info_did).instantiate(
diff --git a/compiler/rustc_hir_typeck/src/closure.rs b/compiler/rustc_hir_typeck/src/closure.rs
index b1cb3ef4d79..459c0498d50 100644
--- a/compiler/rustc_hir_typeck/src/closure.rs
+++ b/compiler/rustc_hir_typeck/src/closure.rs
@@ -142,13 +142,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
                         Ty::new_adt(
                             tcx,
-                            tcx.adt_def(
-                                tcx.require_lang_item(hir::LangItem::Poll, Some(expr_span)),
-                            ),
+                            tcx.adt_def(tcx.require_lang_item(hir::LangItem::Poll, expr_span)),
                             tcx.mk_args(&[Ty::new_adt(
                                 tcx,
                                 tcx.adt_def(
-                                    tcx.require_lang_item(hir::LangItem::Option, Some(expr_span)),
+                                    tcx.require_lang_item(hir::LangItem::Option, expr_span),
                                 ),
                                 tcx.mk_args(&[yield_ty.into()]),
                             )
@@ -204,14 +202,19 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 )
             }
             hir::ClosureKind::CoroutineClosure(kind) => {
-                // async closures always return the type ascribed after the `->` (if present),
-                // and yield `()`.
                 let (bound_return_ty, bound_yield_ty) = match kind {
+                    hir::CoroutineDesugaring::Gen => {
+                        // `iter!` closures always return unit and yield the `Iterator::Item` type
+                        // that we have to infer.
+                        (tcx.types.unit, self.infcx.next_ty_var(expr_span))
+                    }
                     hir::CoroutineDesugaring::Async => {
+                        // async closures always return the type ascribed after the `->` (if present),
+                        // and yield `()`.
                         (bound_sig.skip_binder().output(), tcx.types.unit)
                     }
-                    hir::CoroutineDesugaring::Gen | hir::CoroutineDesugaring::AsyncGen => {
-                        todo!("`gen` and `async gen` closures not supported yet")
+                    hir::CoroutineDesugaring::AsyncGen => {
+                        todo!("`async gen` closures not supported yet")
                     }
                 };
                 // Compute all of the variables that will be used to populate the coroutine.
@@ -465,7 +468,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
             if let Some(trait_def_id) = trait_def_id {
                 let found_kind = match closure_kind {
-                    hir::ClosureKind::Closure => self.tcx.fn_trait_kind_from_def_id(trait_def_id),
+                    hir::ClosureKind::Closure
+                    // FIXME(iter_macro): Someday we'll probably want iterator closures instead of
+                    // just using Fn* for iterators.
+                    | hir::ClosureKind::CoroutineClosure(hir::CoroutineDesugaring::Gen) => {
+                        self.tcx.fn_trait_kind_from_def_id(trait_def_id)
+                    }
                     hir::ClosureKind::CoroutineClosure(hir::CoroutineDesugaring::Async) => self
                         .tcx
                         .async_fn_trait_kind_from_def_id(trait_def_id)
diff --git a/compiler/rustc_hir_typeck/src/coercion.rs b/compiler/rustc_hir_typeck/src/coercion.rs
index d2fc5ae0543..d9fa56fefeb 100644
--- a/compiler/rustc_hir_typeck/src/coercion.rs
+++ b/compiler/rustc_hir_typeck/src/coercion.rs
@@ -47,7 +47,7 @@ use rustc_infer::infer::relate::RelateResult;
 use rustc_infer::infer::{Coercion, DefineOpaqueTypes, InferOk, InferResult};
 use rustc_infer::traits::{
     IfExpressionCause, MatchExpressionArmCause, Obligation, PredicateObligation,
-    PredicateObligations,
+    PredicateObligations, SelectionError,
 };
 use rustc_middle::span_bug;
 use rustc_middle::ty::adjustment::{
@@ -677,7 +677,21 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
                     return Err(TypeError::Mismatch);
                 }
 
-                // Dyn-compatibility violations or miscellaneous.
+                Err(SelectionError::TraitDynIncompatible(_)) => {
+                    // Dyn compatibility errors in coercion will *always* be due to the
+                    // fact that the RHS of the coercion is a non-dyn compatible `dyn Trait`
+                    // writen in source somewhere (otherwise we will never have lowered
+                    // the dyn trait from HIR to middle).
+                    //
+                    // There's no reason to emit yet another dyn compatibility error,
+                    // especially since the span will differ slightly and thus not be
+                    // deduplicated at all!
+                    self.fcx.set_tainted_by_errors(
+                        self.fcx
+                            .dcx()
+                            .span_delayed_bug(self.cause.span, "dyn compatibility during coercion"),
+                    );
+                }
                 Err(err) => {
                     let guar = self.err_ctxt().report_selection_error(
                         obligation.clone(),
@@ -746,8 +760,7 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
                     self.param_env,
                     ty::TraitRef::new(
                         self.tcx,
-                        self.tcx
-                            .require_lang_item(hir::LangItem::PointerLike, Some(self.cause.span)),
+                        self.tcx.require_lang_item(hir::LangItem::PointerLike, self.cause.span),
                         [a],
                     ),
                 ),
@@ -1190,9 +1203,23 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     (ty::FnDef(..), ty::FnDef(..)) => {
                         // Don't reify if the function types have a LUB, i.e., they
                         // are the same function and their parameters have a LUB.
-                        match self
-                            .commit_if_ok(|_| self.at(cause, self.param_env).lub(prev_ty, new_ty))
-                        {
+                        match self.commit_if_ok(|_| {
+                            // We need to eagerly handle nested obligations due to lazy norm.
+                            if self.next_trait_solver() {
+                                let ocx = ObligationCtxt::new(self);
+                                let value = ocx.lub(cause, self.param_env, prev_ty, new_ty)?;
+                                if ocx.select_where_possible().is_empty() {
+                                    Ok(InferOk {
+                                        value,
+                                        obligations: ocx.into_pending_obligations(),
+                                    })
+                                } else {
+                                    Err(TypeError::Mismatch)
+                                }
+                            } else {
+                                self.at(cause, self.param_env).lub(prev_ty, new_ty)
+                            }
+                        }) {
                             // We have a LUB of prev_ty and new_ty, just return it.
                             Ok(ok) => return Ok(self.register_infer_ok_obligations(ok)),
                             Err(_) => {
@@ -1840,17 +1867,16 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
             fcx.err_ctxt().report_mismatched_types(cause, fcx.param_env, expected, found, ty_err);
 
         let due_to_block = matches!(fcx.tcx.hir_node(block_or_return_id), hir::Node::Block(..));
-
-        let parent_id = fcx.tcx.parent_hir_id(block_or_return_id);
-        let parent = fcx.tcx.hir_node(parent_id);
+        let parent = fcx.tcx.parent_hir_node(block_or_return_id);
         if let Some(expr) = expression
             && let hir::Node::Expr(&hir::Expr {
                 kind: hir::ExprKind::Closure(&hir::Closure { body, .. }),
                 ..
             }) = parent
-            && !matches!(fcx.tcx.hir_body(body).value.kind, hir::ExprKind::Block(..))
         {
-            fcx.suggest_missing_semicolon(&mut err, expr, expected, true);
+            let needs_block =
+                !matches!(fcx.tcx.hir_body(body).value.kind, hir::ExprKind::Block(..));
+            fcx.suggest_missing_semicolon(&mut err, expr, expected, needs_block, true);
         }
         // Verify that this is a tail expression of a function, otherwise the
         // label pointing out the cause for the type coercion will be wrong
@@ -1858,7 +1884,7 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
         if let Some(expr) = expression
             && due_to_block
         {
-            fcx.suggest_missing_semicolon(&mut err, expr, expected, false);
+            fcx.suggest_missing_semicolon(&mut err, expr, expected, false, false);
             let pointing_at_return_type = fcx.suggest_mismatched_types_on_tail(
                 &mut err,
                 expr,
@@ -1942,7 +1968,7 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
                 fcx.param_env,
                 ty::TraitRef::new(
                     fcx.tcx,
-                    fcx.tcx.require_lang_item(hir::LangItem::Sized, None),
+                    fcx.tcx.require_lang_item(hir::LangItem::Sized, DUMMY_SP),
                     [sig.output()],
                 ),
             ))
diff --git a/compiler/rustc_hir_typeck/src/demand.rs b/compiler/rustc_hir_typeck/src/demand.rs
index d1bc54ed73e..152c88ad92a 100644
--- a/compiler/rustc_hir_typeck/src/demand.rs
+++ b/compiler/rustc_hir_typeck/src/demand.rs
@@ -84,7 +84,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         self.annotate_expected_due_to_let_ty(err, expr, error);
         self.annotate_loop_expected_due_to_inference(err, expr, error);
-        if self.annotate_mut_binding_to_immutable_binding(err, expr, error) {
+        if self.annotate_mut_binding_to_immutable_binding(err, expr, expr_ty, expected, error) {
             return;
         }
 
@@ -722,8 +722,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     )) => {
                         if let Some(hir::Node::Item(hir::Item {
                             kind:
-                                hir::ItemKind::Static(ident, ty, ..)
-                                | hir::ItemKind::Const(ident, ty, ..),
+                                hir::ItemKind::Static(_, ident, ty, _)
+                                | hir::ItemKind::Const(ident, _, ty, _),
                             ..
                         })) = self.tcx.hir_get_if_local(*def_id)
                         {
@@ -799,17 +799,17 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     /// Detect the following case
     ///
     /// ```text
-    /// fn change_object(mut a: &Ty) {
+    /// fn change_object(mut b: &Ty) {
     ///     let a = Ty::new();
     ///     b = a;
     /// }
     /// ```
     ///
-    /// where the user likely meant to modify the value behind there reference, use `a` as an out
+    /// where the user likely meant to modify the value behind there reference, use `b` as an out
     /// parameter, instead of mutating the local binding. When encountering this we suggest:
     ///
     /// ```text
-    /// fn change_object(a: &'_ mut Ty) {
+    /// fn change_object(b: &'_ mut Ty) {
     ///     let a = Ty::new();
     ///     *b = a;
     /// }
@@ -818,13 +818,15 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         &self,
         err: &mut Diag<'_>,
         expr: &hir::Expr<'_>,
+        expr_ty: Ty<'tcx>,
+        expected: Ty<'tcx>,
         error: Option<TypeError<'tcx>>,
     ) -> bool {
-        if let Some(TypeError::Sorts(ExpectedFound { expected, found })) = error
+        if let Some(TypeError::Sorts(ExpectedFound { .. })) = error
             && let ty::Ref(_, inner, hir::Mutability::Not) = expected.kind()
 
             // The difference between the expected and found values is one level of borrowing.
-            && self.can_eq(self.param_env, *inner, found)
+            && self.can_eq(self.param_env, *inner, expr_ty)
 
             // We have an `ident = expr;` assignment.
             && let hir::Node::Expr(hir::Expr { kind: hir::ExprKind::Assign(lhs, rhs, _), .. }) =
diff --git a/compiler/rustc_hir_typeck/src/errors.rs b/compiler/rustc_hir_typeck/src/errors.rs
index 06103fe1c91..97a90548fc5 100644
--- a/compiler/rustc_hir_typeck/src/errors.rs
+++ b/compiler/rustc_hir_typeck/src/errors.rs
@@ -4,8 +4,8 @@ use std::borrow::Cow;
 
 use rustc_errors::codes::*;
 use rustc_errors::{
-    Applicability, Diag, DiagArgValue, DiagSymbolList, EmissionGuarantee, IntoDiagArg, MultiSpan,
-    Subdiagnostic,
+    Applicability, Diag, DiagArgValue, DiagCtxtHandle, DiagSymbolList, Diagnostic,
+    EmissionGuarantee, IntoDiagArg, Level, MultiSpan, Subdiagnostic,
 };
 use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic};
 use rustc_middle::ty::{self, Ty};
@@ -983,3 +983,55 @@ pub(crate) struct RegisterTypeUnstable<'a> {
     pub span: Span,
     pub ty: Ty<'a>,
 }
+
+#[derive(Diagnostic)]
+#[diag(hir_typeck_naked_asm_outside_naked_fn)]
+pub(crate) struct NakedAsmOutsideNakedFn {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_typeck_no_patterns)]
+pub(crate) struct NoPatterns {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_typeck_params_not_allowed)]
+#[help]
+pub(crate) struct ParamsNotAllowed {
+    #[primary_span]
+    pub span: Span,
+}
+
+pub(crate) struct NakedFunctionsAsmBlock {
+    pub span: Span,
+    pub multiple_asms: Vec<Span>,
+    pub non_asms: Vec<Span>,
+}
+
+impl<G: EmissionGuarantee> Diagnostic<'_, G> for NakedFunctionsAsmBlock {
+    #[track_caller]
+    fn into_diag(self, dcx: DiagCtxtHandle<'_>, level: Level) -> Diag<'_, G> {
+        let mut diag = Diag::new(dcx, level, fluent::hir_typeck_naked_functions_asm_block);
+        diag.span(self.span);
+        diag.code(E0787);
+        for span in self.multiple_asms.iter() {
+            diag.span_label(*span, fluent::hir_typeck_label_multiple_asm);
+        }
+        for span in self.non_asms.iter() {
+            diag.span_label(*span, fluent::hir_typeck_label_non_asm);
+        }
+        diag
+    }
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_typeck_naked_functions_must_naked_asm, code = E0787)]
+pub(crate) struct NakedFunctionsMustNakedAsm {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+}
diff --git a/compiler/rustc_hir_typeck/src/expr.rs b/compiler/rustc_hir_typeck/src/expr.rs
index 2c28ffd1fe3..dfc7935d02b 100644
--- a/compiler/rustc_hir_typeck/src/expr.rs
+++ b/compiler/rustc_hir_typeck/src/expr.rs
@@ -44,9 +44,9 @@ use crate::errors::{
     AddressOfTemporaryTaken, BaseExpressionDoubleDot, BaseExpressionDoubleDotAddExpr,
     BaseExpressionDoubleDotEnableDefaultFieldValues, BaseExpressionDoubleDotRemove,
     CantDereference, FieldMultiplySpecifiedInInitializer, FunctionalRecordUpdateOnNonStruct,
-    HelpUseLatestEdition, NoFieldOnType, NoFieldOnVariant, ReturnLikeStatementKind,
-    ReturnStmtOutsideOfFnBody, StructExprNonExhaustive, TypeMismatchFruTypo,
-    YieldExprOutsideOfCoroutine,
+    HelpUseLatestEdition, NakedAsmOutsideNakedFn, NoFieldOnType, NoFieldOnVariant,
+    ReturnLikeStatementKind, ReturnStmtOutsideOfFnBody, StructExprNonExhaustive,
+    TypeMismatchFruTypo, YieldExprOutsideOfCoroutine,
 };
 use crate::{
     BreakableCtxt, CoroutineTypes, Diverges, FnCtxt, GatherLocalsVisitor, Needs,
@@ -524,7 +524,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             ExprKind::InlineAsm(asm) => {
                 // We defer some asm checks as we may not have resolved the input and output types yet (they may still be infer vars).
                 self.deferred_asm_checks.borrow_mut().push((asm, expr.hir_id));
-                self.check_expr_asm(asm)
+                self.check_expr_asm(asm, expr.span)
             }
             ExprKind::OffsetOf(container, fields) => {
                 self.check_expr_offset_of(container, fields, expr)
@@ -532,14 +532,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             ExprKind::Break(destination, ref expr_opt) => {
                 self.check_expr_break(destination, expr_opt.as_deref(), expr)
             }
-            ExprKind::Continue(destination) => {
-                if destination.target_id.is_ok() {
-                    tcx.types.never
-                } else {
-                    // There was an error; make type-check fail.
-                    Ty::new_misc_error(tcx)
-                }
-            }
+            ExprKind::Continue(destination) => self.check_expr_continue(destination, expr),
             ExprKind::Ret(ref expr_opt) => self.check_expr_return(expr_opt.as_deref(), expr),
             ExprKind::Become(call) => self.check_expr_become(call, expr),
             ExprKind::Let(let_expr) => self.check_expr_let(let_expr, expr.hir_id),
@@ -918,7 +911,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         self,
                         &cause,
                         |mut err| {
-                            self.suggest_missing_semicolon(&mut err, expr, e_ty, false);
+                            self.suggest_missing_semicolon(&mut err, expr, e_ty, false, false);
                             self.suggest_mismatched_types_on_tail(
                                 &mut err, expr, ty, e_ty, target_id,
                             );
@@ -989,6 +982,31 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
     }
 
+    fn check_expr_continue(
+        &self,
+        destination: hir::Destination,
+        expr: &'tcx hir::Expr<'tcx>,
+    ) -> Ty<'tcx> {
+        if let Ok(target_id) = destination.target_id {
+            if let hir::Node::Expr(hir::Expr { kind: ExprKind::Loop(..), .. }) =
+                self.tcx.hir_node(target_id)
+            {
+                self.tcx.types.never
+            } else {
+                // Liveness linting assumes `continue`s all point to loops. We'll report an error
+                // in `check_mod_loops`, but make sure we don't run liveness (#113379, #121623).
+                let guar = self.dcx().span_delayed_bug(
+                    expr.span,
+                    "found `continue` not pointing to loop, but no error reported",
+                );
+                Ty::new_error(self.tcx, guar)
+            }
+        } else {
+            // There was an error; make type-check fail.
+            Ty::new_misc_error(self.tcx)
+        }
+    }
+
     fn check_expr_return(
         &self,
         expr_opt: Option<&'tcx hir::Expr<'tcx>>,
@@ -1389,7 +1407,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 let lhs_deref_ty_is_sized = self
                     .infcx
                     .type_implements_trait(
-                        self.tcx.require_lang_item(LangItem::Sized, None),
+                        self.tcx.require_lang_item(LangItem::Sized, span),
                         [lhs_deref_ty],
                         self.param_env,
                     )
@@ -1775,10 +1793,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let element_ty = if !args.is_empty() {
             let coerce_to = expected
                 .to_option(self)
-                .and_then(|uty| match *self.try_structurally_resolve_type(expr.span, uty).kind() {
-                    ty::Array(ty, _) | ty::Slice(ty) => Some(ty),
-                    _ => None,
-                })
+                .and_then(|uty| self.try_structurally_resolve_type(expr.span, uty).builtin_index())
                 .unwrap_or_else(|| self.next_ty_var(expr.span));
             let mut coerce = CoerceMany::with_coercion_sites(coerce_to, args);
             assert_eq!(self.diverges.get(), Diverges::Maybe);
@@ -1856,10 +1871,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
 
         let uty = match expected {
-            ExpectHasType(uty) => match *uty.kind() {
-                ty::Array(ty, _) | ty::Slice(ty) => Some(ty),
-                _ => None,
-            },
+            ExpectHasType(uty) => uty.builtin_index(),
             _ => None,
         };
 
@@ -1882,62 +1894,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // We defer checking whether the element type is `Copy` as it is possible to have
         // an inference variable as a repeat count and it seems unlikely that `Copy` would
         // have inference side effects required for type checking to succeed.
-        if tcx.features().generic_arg_infer() {
-            self.deferred_repeat_expr_checks.borrow_mut().push((element, element_ty, count));
-        // If the length is 0, we don't create any elements, so we don't copy any.
-        // If the length is 1, we don't copy that one element, we move it. Only check
-        // for `Copy` if the length is larger, or unevaluated.
-        } else if count.try_to_target_usize(self.tcx).is_none_or(|x| x > 1) {
-            self.enforce_repeat_element_needs_copy_bound(element, element_ty);
-        }
+        self.deferred_repeat_expr_checks.borrow_mut().push((element, element_ty, count));
 
         let ty = Ty::new_array_with_const_len(tcx, t, count);
         self.register_wf_obligation(ty.into(), expr.span, ObligationCauseCode::WellFormed(None));
         ty
     }
 
-    /// Requires that `element_ty` is `Copy` (unless it's a const expression itself).
-    pub(super) fn enforce_repeat_element_needs_copy_bound(
-        &self,
-        element: &hir::Expr<'_>,
-        element_ty: Ty<'tcx>,
-    ) {
-        let tcx = self.tcx;
-        // Actual constants as the repeat element get inserted repeatedly instead of getting copied via Copy.
-        match &element.kind {
-            hir::ExprKind::ConstBlock(..) => return,
-            hir::ExprKind::Path(qpath) => {
-                let res = self.typeck_results.borrow().qpath_res(qpath, element.hir_id);
-                if let Res::Def(DefKind::Const | DefKind::AssocConst | DefKind::AnonConst, _) = res
-                {
-                    return;
-                }
-            }
-            _ => {}
-        }
-        // If someone calls a const fn or constructs a const value, they can extract that
-        // out into a separate constant (or a const block in the future), so we check that
-        // to tell them that in the diagnostic. Does not affect typeck.
-        let is_constable = match element.kind {
-            hir::ExprKind::Call(func, _args) => match *self.node_ty(func.hir_id).kind() {
-                ty::FnDef(def_id, _) if tcx.is_stable_const_fn(def_id) => traits::IsConstable::Fn,
-                _ => traits::IsConstable::No,
-            },
-            hir::ExprKind::Path(qpath) => {
-                match self.typeck_results.borrow().qpath_res(&qpath, element.hir_id) {
-                    Res::Def(DefKind::Ctor(_, CtorKind::Const), _) => traits::IsConstable::Ctor,
-                    _ => traits::IsConstable::No,
-                }
-            }
-            _ => traits::IsConstable::No,
-        };
-
-        let lang_item = self.tcx.require_lang_item(LangItem::Copy, None);
-        let code =
-            traits::ObligationCauseCode::RepeatElementCopy { is_constable, elt_span: element.span };
-        self.require_type_meets(element_ty, element.span, code, lang_item);
-    }
-
     fn check_expr_tuple(
         &self,
         elts: &'tcx [hir::Expr<'tcx>],
@@ -3798,7 +3761,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
     }
 
-    fn check_expr_asm(&self, asm: &'tcx hir::InlineAsm<'tcx>) -> Ty<'tcx> {
+    fn check_expr_asm(&self, asm: &'tcx hir::InlineAsm<'tcx>, span: Span) -> Ty<'tcx> {
+        if let rustc_ast::AsmMacro::NakedAsm = asm.asm_macro {
+            if !self.tcx.has_attr(self.body_id, sym::naked) {
+                self.tcx.dcx().emit_err(NakedAsmOutsideNakedFn { span });
+            }
+        }
+
         let mut diverge = asm.asm_macro.diverges(asm.options);
 
         for (op, _op_sp) in asm.operands {
diff --git a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
index 3493d359028..20341318828 100644
--- a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
+++ b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
@@ -13,6 +13,7 @@ use hir::Expr;
 use hir::def::DefKind;
 use hir::pat_util::EnumerateAndAdjustIterator as _;
 use rustc_abi::{FIRST_VARIANT, FieldIdx, VariantIdx};
+use rustc_ast::UnsafeBinderCastKind;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_hir::def::{CtorOf, Res};
 use rustc_hir::def_id::LocalDefId;
@@ -1393,10 +1394,18 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx
                 self.cat_res(expr.hir_id, expr.span, expr_ty, res)
             }
 
-            // both type ascription and unsafe binder casts don't affect
-            // the place-ness of the subexpression.
+            // type ascription doesn't affect the place-ness of the subexpression.
             hir::ExprKind::Type(e, _) => self.cat_expr(e),
-            hir::ExprKind::UnsafeBinderCast(_, e, _) => self.cat_expr(e),
+
+            hir::ExprKind::UnsafeBinderCast(UnsafeBinderCastKind::Unwrap, e, _) => {
+                let base = self.cat_expr(e)?;
+                Ok(self.cat_projection(
+                    expr.hir_id,
+                    base,
+                    expr_ty,
+                    ProjectionKind::UnwrapUnsafeBinder,
+                ))
+            }
 
             hir::ExprKind::AddrOf(..)
             | hir::ExprKind::Call(..)
@@ -1427,6 +1436,7 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx
             | hir::ExprKind::Repeat(..)
             | hir::ExprKind::InlineAsm(..)
             | hir::ExprKind::OffsetOf(..)
+            | hir::ExprKind::UnsafeBinderCast(UnsafeBinderCastKind::Wrap, ..)
             | hir::ExprKind::Err(_) => Ok(self.cat_rvalue(expr.hir_id, expr_ty)),
         }
     }
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs
index 362c7d8efac..8a90e768d70 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs
@@ -409,7 +409,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         code: traits::ObligationCauseCode<'tcx>,
     ) {
         if !ty.references_error() {
-            let lang_item = self.tcx.require_lang_item(LangItem::Sized, None);
+            let lang_item = self.tcx.require_lang_item(LangItem::Sized, span);
             self.require_type_meets(ty, span, code, lang_item);
         }
     }
@@ -443,7 +443,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 // Nothing else is required here.
             } else {
                 // We can't be sure, let's required full `Sized`.
-                let lang_item = self.tcx.require_lang_item(LangItem::Sized, None);
+                let lang_item = self.tcx.require_lang_item(LangItem::Sized, span);
                 self.require_type_meets(ty, span, ObligationCauseCode::Misc, lang_item);
             }
         }
@@ -732,7 +732,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         span: Span,
         hir_id: HirId,
     ) -> (Res, Ty<'tcx>) {
-        let def_id = self.tcx.require_lang_item(lang_item, Some(span));
+        let def_id = self.tcx.require_lang_item(lang_item, span);
         let def_kind = self.tcx.def_kind(def_id);
 
         let item_ty = if let DefKind::Variant = def_kind {
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/adjust_fulfillment_errors.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/adjust_fulfillment_errors.rs
index ee0436f73e1..2bc007b3ad4 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/adjust_fulfillment_errors.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/adjust_fulfillment_errors.rs
@@ -80,12 +80,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let find_param_matching = |matches: &dyn Fn(ty::ParamTerm) -> bool| {
             predicate_args.iter().find_map(|arg| {
                 arg.walk().find_map(|arg| {
-                    if let ty::GenericArgKind::Type(ty) = arg.unpack()
+                    if let ty::GenericArgKind::Type(ty) = arg.kind()
                         && let ty::Param(param_ty) = *ty.kind()
                         && matches(ty::ParamTerm::Ty(param_ty))
                     {
                         Some(arg)
-                    } else if let ty::GenericArgKind::Const(ct) = arg.unpack()
+                    } else if let ty::GenericArgKind::Const(ct) = arg.kind()
                         && let ty::ConstKind::Param(param_ct) = ct.kind()
                         && matches(ty::ParamTerm::Const(param_ct))
                     {
@@ -357,7 +357,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         &self,
         error: &mut traits::FulfillmentError<'tcx>,
         def_id: DefId,
-        param: ty::GenericArg<'tcx>,
+        arg: ty::GenericArg<'tcx>,
         qpath: &hir::QPath<'tcx>,
     ) -> bool {
         match qpath {
@@ -365,7 +365,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 for segment in path.segments.iter().rev() {
                     if let Res::Def(kind, def_id) = segment.res
                         && !matches!(kind, DefKind::Mod | DefKind::ForeignMod)
-                        && self.point_at_generic_if_possible(error, def_id, param, segment)
+                        && self.point_at_generic_if_possible(error, def_id, arg, segment)
                     {
                         return true;
                     }
@@ -373,7 +373,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 // Handle `Self` param specifically, since it's separated in
                 // the path representation
                 if let Some(self_ty) = self_ty
-                    && let ty::GenericArgKind::Type(ty) = param.unpack()
+                    && let ty::GenericArgKind::Type(ty) = arg.kind()
                     && ty == self.tcx.types.self_param
                 {
                     error.obligation.cause.span = self_ty
@@ -384,12 +384,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 }
             }
             hir::QPath::TypeRelative(self_ty, segment) => {
-                if self.point_at_generic_if_possible(error, def_id, param, segment) {
+                if self.point_at_generic_if_possible(error, def_id, arg, segment) {
                     return true;
                 }
                 // Handle `Self` param specifically, since it's separated in
                 // the path representation
-                if let ty::GenericArgKind::Type(ty) = param.unpack()
+                if let ty::GenericArgKind::Type(ty) = arg.kind()
                     && ty == self.tcx.types.self_param
                 {
                     error.obligation.cause.span = self_ty
@@ -424,10 +424,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // the args list does not, then we should chop off all of the lifetimes,
         // since they're all elided.
         let segment_args = segment.args().args;
-        if matches!(own_args[0].unpack(), ty::GenericArgKind::Lifetime(_))
+        if matches!(own_args[0].kind(), ty::GenericArgKind::Lifetime(_))
             && segment_args.first().is_some_and(|arg| arg.is_ty_or_const())
             && let Some(offset) = own_args.iter().position(|arg| {
-                matches!(arg.unpack(), ty::GenericArgKind::Type(_) | ty::GenericArgKind::Const(_))
+                matches!(arg.kind(), ty::GenericArgKind::Type(_) | ty::GenericArgKind::Const(_))
             })
             && let Some(new_index) = index.checked_sub(offset)
         {
@@ -750,7 +750,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             return Ok(expr);
         }
 
-        let ty::GenericArgKind::Type(in_ty) = in_ty.unpack() else {
+        let ty::GenericArgKind::Type(in_ty) = in_ty.kind() else {
             return Err(expr);
         };
 
@@ -1045,7 +1045,7 @@ fn find_param_in_ty<'tcx>(
         if arg == param_to_point_at {
             return true;
         }
-        if let ty::GenericArgKind::Type(ty) = arg.unpack()
+        if let ty::GenericArgKind::Type(ty) = arg.kind()
             && let ty::Alias(ty::Projection | ty::Inherent, ..) = ty.kind()
         {
             // This logic may seem a bit strange, but typically when
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs
index d2cdfe22a3a..95c7f251c88 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs
@@ -4,12 +4,12 @@ use itertools::Itertools;
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_errors::codes::*;
 use rustc_errors::{Applicability, Diag, ErrorGuaranteed, MultiSpan, a_or_an, listify, pluralize};
-use rustc_hir::def::{CtorOf, DefKind, Res};
+use rustc_hir::def::{CtorKind, CtorOf, DefKind, Res};
 use rustc_hir::def_id::DefId;
 use rustc_hir::intravisit::Visitor;
-use rustc_hir::{ExprKind, HirId, Node, QPath};
+use rustc_hir::{ExprKind, HirId, LangItem, Node, QPath};
 use rustc_hir_analysis::check::potentially_plural_count;
-use rustc_hir_analysis::hir_ty_lowering::HirTyLowerer;
+use rustc_hir_analysis::hir_ty_lowering::{HirTyLowerer, PermitVariants};
 use rustc_index::IndexVec;
 use rustc_infer::infer::{DefineOpaqueTypes, InferOk, TypeTrace};
 use rustc_middle::ty::adjustment::AllowTwoPhase;
@@ -104,24 +104,96 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     pub(in super::super) fn check_repeat_exprs(&self) {
         let mut deferred_repeat_expr_checks = self.deferred_repeat_expr_checks.borrow_mut();
         debug!("FnCtxt::check_repeat_exprs: {} deferred checks", deferred_repeat_expr_checks.len());
-        for (element, element_ty, count) in deferred_repeat_expr_checks.drain(..) {
-            // We want to emit an error if the const is not structurally resolveable as otherwise
-            // we can find up conservatively proving `Copy` which may infer the repeat expr count
-            // to something that never required `Copy` in the first place.
-            let count =
-                self.structurally_resolve_const(element.span, self.normalize(element.span, count));
-
-            // Avoid run on "`NotCopy: Copy` is not implemented" errors when the repeat expr count
-            // is erroneous/unknown. The user might wind up specifying a repeat count of 0/1.
-            if count.references_error() {
-                continue;
-            }
 
-            // If the length is 0, we don't create any elements, so we don't copy any.
-            // If the length is 1, we don't copy that one element, we move it. Only check
-            // for `Copy` if the length is larger.
-            if count.try_to_target_usize(self.tcx).is_none_or(|x| x > 1) {
-                self.enforce_repeat_element_needs_copy_bound(element, element_ty);
+        let deferred_repeat_expr_checks = deferred_repeat_expr_checks
+            .drain(..)
+            .flat_map(|(element, element_ty, count)| {
+                // Actual constants as the repeat element are inserted repeatedly instead
+                // of being copied via `Copy`, so we don't need to attempt to structurally
+                // resolve the repeat count which may unnecessarily error.
+                match &element.kind {
+                    hir::ExprKind::ConstBlock(..) => return None,
+                    hir::ExprKind::Path(qpath) => {
+                        let res = self.typeck_results.borrow().qpath_res(qpath, element.hir_id);
+                        if let Res::Def(DefKind::Const | DefKind::AssocConst, _) = res {
+                            return None;
+                        }
+                    }
+                    _ => {}
+                }
+
+                // We want to emit an error if the const is not structurally resolveable
+                // as otherwise we can wind up conservatively proving `Copy` which may
+                // infer the repeat expr count to something that never required `Copy` in
+                // the first place.
+                let count = self
+                    .structurally_resolve_const(element.span, self.normalize(element.span, count));
+
+                // Avoid run on "`NotCopy: Copy` is not implemented" errors when the
+                // repeat expr count is erroneous/unknown. The user might wind up
+                // specifying a repeat count of 0/1.
+                if count.references_error() {
+                    return None;
+                }
+
+                Some((element, element_ty, count))
+            })
+            // We collect to force the side effects of structurally resolving the repeat
+            // count to happen in one go, to avoid side effects from proving `Copy`
+            // affecting whether repeat counts are known or not. If we did not do this we
+            // would get results that depend on the order that we evaluate each repeat
+            // expr's `Copy` check.
+            .collect::<Vec<_>>();
+
+        let enforce_copy_bound = |element: &hir::Expr<'_>, element_ty| {
+            // If someone calls a const fn or constructs a const value, they can extract that
+            // out into a separate constant (or a const block in the future), so we check that
+            // to tell them that in the diagnostic. Does not affect typeck.
+            let is_constable = match element.kind {
+                hir::ExprKind::Call(func, _args) => match *self.node_ty(func.hir_id).kind() {
+                    ty::FnDef(def_id, _) if self.tcx.is_stable_const_fn(def_id) => {
+                        traits::IsConstable::Fn
+                    }
+                    _ => traits::IsConstable::No,
+                },
+                hir::ExprKind::Path(qpath) => {
+                    match self.typeck_results.borrow().qpath_res(&qpath, element.hir_id) {
+                        Res::Def(DefKind::Ctor(_, CtorKind::Const), _) => traits::IsConstable::Ctor,
+                        _ => traits::IsConstable::No,
+                    }
+                }
+                _ => traits::IsConstable::No,
+            };
+
+            let lang_item = self.tcx.require_lang_item(LangItem::Copy, element.span);
+            let code = traits::ObligationCauseCode::RepeatElementCopy {
+                is_constable,
+                elt_span: element.span,
+            };
+            self.require_type_meets(element_ty, element.span, code, lang_item);
+        };
+
+        for (element, element_ty, count) in deferred_repeat_expr_checks {
+            match count.kind() {
+                ty::ConstKind::Value(val) => {
+                    if val.try_to_target_usize(self.tcx).is_none_or(|count| count > 1) {
+                        enforce_copy_bound(element, element_ty)
+                    } else {
+                        // If the length is 0 or 1 we don't actually copy the element, we either don't create it
+                        // or we just use the one value.
+                    }
+                }
+
+                // If the length is a generic parameter or some rigid alias then conservatively
+                // require `element_ty: Copy` as it may wind up being `>1` after monomorphization.
+                ty::ConstKind::Param(_)
+                | ty::ConstKind::Expr(_)
+                | ty::ConstKind::Placeholder(_)
+                | ty::ConstKind::Unevaluated(_) => enforce_copy_bound(element, element_ty),
+
+                ty::ConstKind::Bound(_, _) | ty::ConstKind::Infer(_) | ty::ConstKind::Error(_) => {
+                    unreachable!()
+                }
             }
         }
     }
@@ -1484,25 +1556,20 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             SuggestionText::Reorder => Some("reorder these arguments".to_string()),
             SuggestionText::DidYouMean => Some("did you mean".to_string()),
         };
-        if let Some(suggestion_text) = suggestion_text {
+        if let Some(suggestion_text) = suggestion_text
+            && !full_call_span.in_external_macro(self.sess().source_map())
+        {
             let source_map = self.sess().source_map();
-            let (mut suggestion, suggestion_span) = if let Some(call_span) =
-                full_call_span.find_ancestor_inside_same_ctxt(error_span)
-            {
-                ("(".to_string(), call_span.shrink_to_hi().to(error_span.shrink_to_hi()))
+            let suggestion_span = if let Some(args_span) = error_span.trim_start(full_call_span) {
+                // Span of the braces, e.g. `(a, b, c)`.
+                args_span
             } else {
-                (
-                    format!(
-                        "{}(",
-                        source_map.span_to_snippet(full_call_span).unwrap_or_else(|_| {
-                            fn_def_id.map_or("".to_string(), |fn_def_id| {
-                                tcx.item_name(fn_def_id).to_string()
-                            })
-                        })
-                    ),
-                    error_span,
-                )
+                // The arg span of a function call that wasn't even given braces
+                // like what might happen with delegation reuse.
+                // e.g. `reuse HasSelf::method;` should suggest `reuse HasSelf::method($args);`.
+                full_call_span.shrink_to_hi()
             };
+            let mut suggestion = "(".to_owned();
             let mut needs_comma = false;
             for (expected_idx, provided_idx) in matched_inputs.iter_enumerated() {
                 if needs_comma {
@@ -1613,8 +1680,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             ast::LitKind::CStr(_, _) => Ty::new_imm_ref(
                 tcx,
                 tcx.lifetimes.re_static,
-                tcx.type_of(tcx.require_lang_item(hir::LangItem::CStr, Some(lit.span)))
-                    .skip_binder(),
+                tcx.type_of(tcx.require_lang_item(hir::LangItem::CStr, lit.span)).skip_binder(),
             ),
             ast::LitKind::Err(guar) => Ty::new_error(tcx, guar),
         }
@@ -2105,15 +2171,25 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         match *qpath {
             QPath::Resolved(ref maybe_qself, path) => {
                 let self_ty = maybe_qself.as_ref().map(|qself| self.lower_ty(qself).raw);
-                let ty = self.lowerer().lower_path(self_ty, path, hir_id, true);
+                let ty = self.lowerer().lower_resolved_ty_path(
+                    self_ty,
+                    path,
+                    hir_id,
+                    PermitVariants::Yes,
+                );
                 (path.res, LoweredTy::from_raw(self, path_span, ty))
             }
-            QPath::TypeRelative(qself, segment) => {
-                let ty = self.lower_ty(qself);
+            QPath::TypeRelative(hir_self_ty, segment) => {
+                let self_ty = self.lower_ty(hir_self_ty);
 
-                let result = self
-                    .lowerer()
-                    .lower_assoc_path_ty(hir_id, path_span, ty.raw, qself, segment, true);
+                let result = self.lowerer().lower_type_relative_ty_path(
+                    self_ty.raw,
+                    hir_self_ty,
+                    segment,
+                    hir_id,
+                    path_span,
+                    PermitVariants::Yes,
+                );
                 let ty = result
                     .map(|(ty, _, _)| ty)
                     .unwrap_or_else(|guar| Ty::new_error(self.tcx(), guar));
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/inspect_obligations.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/inspect_obligations.rs
index e068e607902..7f1f3c3c802 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/inspect_obligations.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/inspect_obligations.rs
@@ -4,6 +4,7 @@ use rustc_infer::traits::{self, ObligationCause, PredicateObligations};
 use rustc_middle::traits::solve::GoalSource;
 use rustc_middle::ty::{self, Ty, TypeVisitableExt};
 use rustc_span::Span;
+use rustc_trait_selection::solve::Certainty;
 use rustc_trait_selection::solve::inspect::{
     InspectConfig, InspectGoal, ProofTreeInferCtxtExt, ProofTreeVisitor,
 };
@@ -117,6 +118,20 @@ impl<'a, 'tcx> ProofTreeVisitor<'tcx> for NestedObligationsForSelfTy<'a, 'tcx> {
     }
 
     fn visit_goal(&mut self, inspect_goal: &InspectGoal<'_, 'tcx>) {
+        // No need to walk into goal subtrees that certainly hold, since they
+        // wouldn't then be stalled on an infer var.
+        // FIXME: We also walk into normalizes-to goals since their certainty
+        // is forced to `Certainty::Yes` since they pass down ambiguous subgoals
+        // to their parent.
+        if inspect_goal.result() == Ok(Certainty::Yes)
+            && !matches!(
+                inspect_goal.goal().predicate.kind().skip_binder(),
+                ty::PredicateKind::NormalizesTo(_)
+            )
+        {
+            return;
+        }
+
         let tcx = self.fcx.tcx;
         let goal = inspect_goal.goal();
         if self.fcx.predicate_has_self_ty(goal.predicate, self.self_ty)
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs
index de189b30109..a5c0829b8d9 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs
@@ -14,7 +14,7 @@ use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::{self as hir, HirId, ItemLocalMap};
 use rustc_hir_analysis::hir_ty_lowering::{HirTyLowerer, RegionInferReason};
 use rustc_infer::infer;
-use rustc_infer::traits::Obligation;
+use rustc_infer::traits::{DynCompatibilityViolation, Obligation};
 use rustc_middle::ty::{self, Const, Ty, TyCtxt, TypeVisitableExt};
 use rustc_session::Session;
 use rustc_span::{self, DUMMY_SP, ErrorGuaranteed, Ident, Span, sym};
@@ -81,7 +81,7 @@ pub(crate) struct FnCtxt<'a, 'tcx> {
     /// you get indicates whether any subexpression that was
     /// evaluating up to and including `X` diverged.
     ///
-    /// We currently use this flag only for diagnostic purposes:
+    /// We currently use this flag for the following purposes:
     ///
     /// - To warn about unreachable code: if, after processing a
     ///   sub-expression but before we have applied the effects of the
@@ -94,6 +94,8 @@ pub(crate) struct FnCtxt<'a, 'tcx> {
     ///   warning. This corresponds to something like `{return;
     ///   foo();}` or `{return; 22}`, where we would warn on the
     ///   `foo()` or `22`.
+    /// - To assign the `!` type to block expressions with diverging
+    ///   statements.
     ///
     /// An expression represents dead code if, after checking it,
     /// the diverges flag is set to something other than `Maybe`.
@@ -308,17 +310,16 @@ impl<'tcx> HirTyLowerer<'tcx> for FnCtxt<'_, 'tcx> {
         ))
     }
 
-    fn lower_assoc_shared(
+    fn lower_assoc_item_path(
         &self,
         span: Span,
         item_def_id: DefId,
         item_segment: &rustc_hir::PathSegment<'tcx>,
         poly_trait_ref: ty::PolyTraitRef<'tcx>,
-        _assoc_tag: ty::AssocTag,
     ) -> Result<(DefId, ty::GenericArgsRef<'tcx>), ErrorGuaranteed> {
         let trait_ref = self.instantiate_binder_with_fresh_vars(
             span,
-            // FIXME(mgca): this should be assoc const if that is the `kind`
+            // FIXME(mgca): `item_def_id` can be an AssocConst; rename this variant.
             infer::BoundRegionConversionTime::AssocTypeProjection(item_def_id),
             poly_trait_ref,
         );
@@ -389,6 +390,10 @@ impl<'tcx> HirTyLowerer<'tcx> for FnCtxt<'_, 'tcx> {
         };
         (input_tys, output_ty)
     }
+
+    fn dyn_compatibility_violations(&self, trait_def_id: DefId) -> Vec<DynCompatibilityViolation> {
+        self.tcx.dyn_compatibility_violations(trait_def_id).to_vec()
+    }
 }
 
 /// The `ty` representation of a user-provided type. Depending on the use-site
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
index 1079262b5af..66af085cfd4 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
@@ -764,6 +764,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         expression: &'tcx hir::Expr<'tcx>,
         expected: Ty<'tcx>,
         needs_block: bool,
+        parent_is_closure: bool,
     ) {
         if expected.is_unit() {
             // `BlockTailExpression` only relevant if the tail expr would be
@@ -799,6 +800,17 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         );
                     }
                 }
+                ExprKind::Path(..) | ExprKind::Lit(_)
+                    if parent_is_closure
+                        && !expression.span.in_external_macro(self.tcx.sess.source_map()) =>
+                {
+                    err.span_suggestion_verbose(
+                        expression.span.shrink_to_lo(),
+                        "consider ignoring the value",
+                        "_ = ",
+                        Applicability::MachineApplicable,
+                    );
+                }
                 _ => (),
             }
         }
@@ -2667,7 +2679,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     let mut sugg_sp = sp;
                     if let hir::ExprKind::MethodCall(segment, receiver, args, _) = expr.kind {
                         let clone_trait =
-                            self.tcx.require_lang_item(LangItem::Clone, Some(segment.ident.span));
+                            self.tcx.require_lang_item(LangItem::Clone, segment.ident.span);
                         if args.is_empty()
                             && self
                                 .typeck_results
@@ -2701,6 +2713,31 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         ));
                     }
 
+                    // Don't try to suggest ref/deref on an `if` expression, because:
+                    // - The `if` could be part of a desugared `if else` statement,
+                    //   which would create impossible suggestions such as `if ... { ... } else &if { ... } else { ... }`.
+                    // - In general the suggestions it creates such as `&if ... { ... } else { ... }` are not very helpful.
+                    // We try to generate a suggestion such as `if ... { &... } else { &... }` instead.
+                    if let hir::ExprKind::If(_c, then, els) = expr.kind {
+                        // The `then` of a `Expr::If` always contains a block, and that block may have a final expression that we can borrow
+                        // If the block does not have a final expression, it will return () and we do not make a suggestion to borrow that.
+                        let ExprKind::Block(then, _) = then.kind else { return None };
+                        let Some(then) = then.expr else { return None };
+                        let (mut suggs, help, app, verbose, mutref) =
+                            self.suggest_deref_or_ref(then, checked_ty, expected)?;
+
+                        // If there is no `else`, the return type of this `if` will be (), so suggesting to change the `then` block is useless
+                        let els_expr = match els?.kind {
+                            ExprKind::Block(block, _) => block.expr?,
+                            _ => els?,
+                        };
+                        let (else_suggs, ..) =
+                            self.suggest_deref_or_ref(els_expr, checked_ty, expected)?;
+                        suggs.extend(else_suggs);
+
+                        return Some((suggs, help, app, verbose, mutref));
+                    }
+
                     if let Some((sugg, msg)) = self.can_use_as_ref(expr) {
                         return Some((
                             sugg,
diff --git a/compiler/rustc_hir_typeck/src/gather_locals.rs b/compiler/rustc_hir_typeck/src/gather_locals.rs
index 956671fc66e..7d99b0e7869 100644
--- a/compiler/rustc_hir_typeck/src/gather_locals.rs
+++ b/compiler/rustc_hir_typeck/src/gather_locals.rs
@@ -218,7 +218,12 @@ impl<'a, 'tcx> Visitor<'tcx> for GatherLocalsVisitor<'a, 'tcx> {
             );
         }
         let old_outermost_fn_param_pat = self.outermost_fn_param_pat.take();
-        intravisit::walk_pat(self, p);
+        if let PatKind::Guard(subpat, _) = p.kind {
+            // We'll visit the guard when checking it. Don't gather its locals twice.
+            self.visit_pat(subpat);
+        } else {
+            intravisit::walk_pat(self, p);
+        }
         self.outermost_fn_param_pat = old_outermost_fn_param_pat;
     }
 
diff --git a/compiler/rustc_hir_typeck/src/inline_asm.rs b/compiler/rustc_hir_typeck/src/inline_asm.rs
index 6399f0a78ae..b59c1752c25 100644
--- a/compiler/rustc_hir_typeck/src/inline_asm.rs
+++ b/compiler/rustc_hir_typeck/src/inline_asm.rs
@@ -171,7 +171,7 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
             _ if ty.references_error() => return None,
             ty::Adt(adt, args) if self.tcx().is_lang_item(adt.did(), LangItem::MaybeUninit) => {
                 let fields = &adt.non_enum_variant().fields;
-                let ty = fields[FieldIdx::from_u32(1)].ty(self.tcx(), args);
+                let ty = fields[FieldIdx::ONE].ty(self.tcx(), args);
                 // FIXME: Are we just trying to map to the `T` in `MaybeUninit<T>`?
                 // If so, just get it from the args.
                 let ty::Adt(ty, args) = ty.kind() else {
diff --git a/compiler/rustc_hir_typeck/src/lib.rs b/compiler/rustc_hir_typeck/src/lib.rs
index 161f5e981d4..741a616631a 100644
--- a/compiler/rustc_hir_typeck/src/lib.rs
+++ b/compiler/rustc_hir_typeck/src/lib.rs
@@ -1,12 +1,11 @@
 // tidy-alphabetical-start
 #![allow(rustc::diagnostic_outside_of_impl)]
 #![allow(rustc::untranslatable_diagnostic)]
-#![feature(array_windows)]
+#![feature(assert_matches)]
 #![feature(box_patterns)]
 #![feature(if_let_guard)]
 #![feature(iter_intersperse)]
 #![feature(never_type)]
-#![feature(try_blocks)]
 // tidy-alphabetical-end
 
 mod _match;
@@ -30,6 +29,7 @@ mod fn_ctxt;
 mod gather_locals;
 mod intrinsicck;
 mod method;
+mod naked_functions;
 mod op;
 mod opaque_types;
 mod pat;
@@ -54,8 +54,8 @@ use rustc_middle::query::Providers;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_middle::{bug, span_bug};
 use rustc_session::config;
-use rustc_span::Span;
 use rustc_span::def_id::LocalDefId;
+use rustc_span::{Span, sym};
 use tracing::{debug, instrument};
 use typeck_root_ctxt::TypeckRootCtxt;
 
@@ -148,7 +148,7 @@ fn typeck_with_inspect<'tcx>(
             tcx.fn_sig(def_id).instantiate_identity()
         };
 
-        check_abi(tcx, span, fn_sig.abi());
+        check_abi(tcx, id, span, fn_sig.abi());
 
         // Compute the function signature from point of view of inside the fn.
         let mut fn_sig = tcx.liberate_late_bound_regions(def_id.to_def_id(), fn_sig);
@@ -169,6 +169,10 @@ fn typeck_with_inspect<'tcx>(
                 .map(|(idx, ty)| fcx.normalize(arg_span(idx), ty)),
         );
 
+        if tcx.has_attr(def_id, sym::naked) {
+            naked_functions::typeck_naked_fn(tcx, def_id, body);
+        }
+
         check_fn(&mut fcx, fn_sig, None, decl, def_id, body, tcx.features().unsized_fn_params());
     } else {
         let expected_type = if let Some(infer_ty) = infer_type_if_missing(&fcx, node) {
@@ -195,13 +199,16 @@ fn typeck_with_inspect<'tcx>(
         fcx.write_ty(id, expected_type);
     };
 
-    // Whether to check repeat exprs before/after inference fallback is somewhat arbitrary of a decision
-    // as neither option is strictly more permissive than the other. However, we opt to check repeat exprs
-    // first as errors from not having inferred array lengths yet seem less confusing than errors from inference
-    // fallback arbitrarily inferring something incompatible with `Copy` inference side effects.
+    // Whether to check repeat exprs before/after inference fallback is somewhat
+    // arbitrary of a decision as neither option is strictly more permissive than
+    // the other. However, we opt to check repeat exprs first as errors from not
+    // having inferred array lengths yet seem less confusing than errors from inference
+    // fallback arbitrarily inferring something incompatible with `Copy` inference
+    // side effects.
     //
-    // This should also be forwards compatible with moving repeat expr checks to a custom goal kind or using
-    // marker traits in the future.
+    // FIXME(#140855): This should also be forwards compatible with moving
+    // repeat expr checks to a custom goal kind or using marker traits in
+    // the future.
     fcx.check_repeat_exprs();
 
     fcx.type_inference_fallback();
diff --git a/compiler/rustc_hir_typeck/src/method/confirm.rs b/compiler/rustc_hir_typeck/src/method/confirm.rs
index a614b4f00ff..53b5dff9c6b 100644
--- a/compiler/rustc_hir_typeck/src/method/confirm.rs
+++ b/compiler/rustc_hir_typeck/src/method/confirm.rs
@@ -291,6 +291,14 @@ impl<'a, 'tcx> ConfirmContext<'a, 'tcx> {
             probe::ObjectPick => {
                 let trait_def_id = pick.item.container_id(self.tcx);
 
+                // If the trait is not object safe (specifically, we care about when
+                // the receiver is not valid), then there's a chance that we will not
+                // actually be able to recover the object by derefing the receiver like
+                // we should if it were valid.
+                if !self.tcx.is_dyn_compatible(trait_def_id) {
+                    return ty::GenericArgs::extend_with_error(self.tcx, trait_def_id, &[]);
+                }
+
                 // This shouldn't happen for non-region error kinds, but may occur
                 // when we have error regions. Specifically, since we canonicalize
                 // during method steps, we may successfully deref when we assemble
diff --git a/compiler/rustc_hir_typeck/src/method/probe.rs b/compiler/rustc_hir_typeck/src/method/probe.rs
index bda051f1560..725240b480b 100644
--- a/compiler/rustc_hir_typeck/src/method/probe.rs
+++ b/compiler/rustc_hir_typeck/src/method/probe.rs
@@ -1,7 +1,9 @@
+use std::assert_matches::debug_assert_matches;
 use std::cell::{Cell, RefCell};
 use std::cmp::max;
 use std::ops::Deref;
 
+use rustc_attr_parsing::is_doc_alias_attrs_contain_symbol;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::sso::SsoHashSet;
 use rustc_errors::Applicability;
@@ -15,7 +17,7 @@ use rustc_infer::traits::ObligationCauseCode;
 use rustc_middle::middle::stability;
 use rustc_middle::query::Providers;
 use rustc_middle::ty::elaborate::supertrait_def_ids;
-use rustc_middle::ty::fast_reject::{TreatParams, simplify_type};
+use rustc_middle::ty::fast_reject::{DeepRejectCtxt, TreatParams, simplify_type};
 use rustc_middle::ty::{
     self, AssocItem, AssocItemContainer, GenericArgs, GenericArgsRef, GenericParamDefKind,
     ParamEnvAnd, Ty, TyCtxt, TypeVisitableExt, Upcast,
@@ -806,8 +808,8 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                     );
                 }
             }
-            ty::Param(p) => {
-                self.assemble_inherent_candidates_from_param(p);
+            ty::Param(_) => {
+                self.assemble_inherent_candidates_from_param(raw_self_ty);
             }
             ty::Bool
             | ty::Char
@@ -908,18 +910,16 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
     }
 
     #[instrument(level = "debug", skip(self))]
-    fn assemble_inherent_candidates_from_param(&mut self, param_ty: ty::ParamTy) {
+    fn assemble_inherent_candidates_from_param(&mut self, param_ty: Ty<'tcx>) {
+        debug_assert_matches!(param_ty.kind(), ty::Param(_));
+
+        let tcx = self.tcx;
         let bounds = self.param_env.caller_bounds().iter().filter_map(|predicate| {
             let bound_predicate = predicate.kind();
             match bound_predicate.skip_binder() {
-                ty::ClauseKind::Trait(trait_predicate) => {
-                    match *trait_predicate.trait_ref.self_ty().kind() {
-                        ty::Param(p) if p == param_ty => {
-                            Some(bound_predicate.rebind(trait_predicate.trait_ref))
-                        }
-                        _ => None,
-                    }
-                }
+                ty::ClauseKind::Trait(trait_predicate) => DeepRejectCtxt::relate_rigid_rigid(tcx)
+                    .types_may_unify(param_ty, trait_predicate.trait_ref.self_ty())
+                    .then(|| bound_predicate.rebind(trait_predicate.trait_ref)),
                 ty::ClauseKind::RegionOutlives(_)
                 | ty::ClauseKind::TypeOutlives(_)
                 | ty::ClauseKind::Projection(_)
@@ -2333,10 +2333,13 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
         };
         let hir_id = self.fcx.tcx.local_def_id_to_hir_id(local_def_id);
         let attrs = self.fcx.tcx.hir_attrs(hir_id);
+
+        if is_doc_alias_attrs_contain_symbol(attrs.into_iter(), method.name) {
+            return true;
+        }
+
         for attr in attrs {
-            if attr.has_name(sym::doc) {
-                // do nothing
-            } else if attr.has_name(sym::rustc_confusables) {
+            if attr.has_name(sym::rustc_confusables) {
                 let Some(confusables) = attr.meta_item_list() else {
                     continue;
                 };
@@ -2348,33 +2351,6 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                         return true;
                     }
                 }
-                continue;
-            } else {
-                continue;
-            };
-            let Some(values) = attr.meta_item_list() else {
-                continue;
-            };
-            for v in values {
-                if !v.has_name(sym::alias) {
-                    continue;
-                }
-                if let Some(nested) = v.meta_item_list() {
-                    // #[doc(alias("foo", "bar"))]
-                    for n in nested {
-                        if let Some(lit) = n.lit()
-                            && method.name == lit.symbol
-                        {
-                            return true;
-                        }
-                    }
-                } else if let Some(meta) = v.meta_item()
-                    && let Some(lit) = meta.name_value_literal()
-                    && method.name == lit.symbol
-                {
-                    // #[doc(alias = "foo")]
-                    return true;
-                }
             }
         }
         false
diff --git a/compiler/rustc_hir_typeck/src/method/suggest.rs b/compiler/rustc_hir_typeck/src/method/suggest.rs
index 342eed751a5..2fac13b7201 100644
--- a/compiler/rustc_hir_typeck/src/method/suggest.rs
+++ b/compiler/rustc_hir_typeck/src/method/suggest.rs
@@ -599,7 +599,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let tcx = self.tcx;
         let rcvr_ty = self.resolve_vars_if_possible(rcvr_ty);
         let mut ty_file = None;
-        let (mut ty_str, short_ty_str) =
+        let (ty_str, short_ty_str) =
             if trait_missing_method && let ty::Dynamic(predicates, _, _) = rcvr_ty.kind() {
                 (predicates.to_string(), with_forced_trimmed_paths!(predicates.to_string()))
             } else {
@@ -738,10 +738,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             err.span_label(within_macro_span, "due to this macro variable");
         }
 
-        if short_ty_str.len() < ty_str.len() && ty_str.len() > 10 {
-            ty_str = short_ty_str;
-        }
-
         if rcvr_ty.references_error() {
             err.downgrade_to_delayed_bug();
         }
@@ -2230,7 +2226,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 let infer_args = self.tcx.mk_args_from_iter(args.into_iter().map(|arg| {
                     if !arg.is_suggestable(self.tcx, true) {
                         has_unsuggestable_args = true;
-                        match arg.unpack() {
+                        match arg.kind() {
                             GenericArgKind::Lifetime(_) => self
                                 .next_region_var(RegionVariableOrigin::MiscVariable(DUMMY_SP))
                                 .into(),
@@ -2847,7 +2843,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 let [first] = ***args else {
                     return;
                 };
-                let ty::GenericArgKind::Type(ty) = first.unpack() else {
+                let ty::GenericArgKind::Type(ty) = first.kind() else {
                     return;
                 };
                 let Ok(pick) = self.lookup_probe_for_diagnostic(
@@ -3498,7 +3494,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             continue;
                         }
                         trait_in_other_version_found = self
-                            .detect_and_explain_multiple_crate_versions(
+                            .detect_and_explain_multiple_crate_versions_of_trait_item(
                                 err,
                                 pick.item.def_id,
                                 rcvr.hir_id,
@@ -3705,12 +3701,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // same crate.
 
             let rcvr_ty = self.node_ty_opt(ty.hir_id);
-            trait_in_other_version_found = self.detect_and_explain_multiple_crate_versions(
-                err,
-                assoc.def_id,
-                ty.hir_id,
-                rcvr_ty,
-            );
+            trait_in_other_version_found = self
+                .detect_and_explain_multiple_crate_versions_of_trait_item(
+                    err,
+                    assoc.def_id,
+                    ty.hir_id,
+                    rcvr_ty,
+                );
         }
         if !trait_in_other_version_found
             && self.suggest_valid_traits(err, item_name, valid_out_of_scope_traits, true)
@@ -4102,7 +4099,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
     }
 
-    fn detect_and_explain_multiple_crate_versions(
+    fn detect_and_explain_multiple_crate_versions_of_trait_item(
         &self,
         err: &mut Diag<'_>,
         item_def_id: DefId,
@@ -4115,6 +4112,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             return false;
         }
         let trait_def_id = self.tcx.parent(item_def_id);
+        if !self.tcx.is_trait(trait_def_id) {
+            return false;
+        }
         let krate = self.tcx.crate_name(trait_def_id.krate);
         let name = self.tcx.item_name(trait_def_id);
         let candidates: Vec<_> = traits
diff --git a/compiler/rustc_passes/src/naked_functions.rs b/compiler/rustc_hir_typeck/src/naked_functions.rs
index 3c9f8b72c36..2518d6478e6 100644
--- a/compiler/rustc_passes/src/naked_functions.rs
+++ b/compiler/rustc_hir_typeck/src/naked_functions.rs
@@ -1,56 +1,29 @@
 //! Checks validity of naked functions.
 
 use rustc_hir as hir;
-use rustc_hir::def::DefKind;
-use rustc_hir::def_id::{LocalDefId, LocalModDefId};
+use rustc_hir::def_id::LocalDefId;
 use rustc_hir::intravisit::Visitor;
 use rustc_hir::{ExprKind, HirIdSet, StmtKind};
-use rustc_middle::hir::nested_filter::OnlyBodies;
-use rustc_middle::query::Providers;
 use rustc_middle::span_bug;
 use rustc_middle::ty::TyCtxt;
 use rustc_span::{Span, sym};
 
 use crate::errors::{
-    NakedAsmOutsideNakedFn, NakedFunctionsAsmBlock, NakedFunctionsMustNakedAsm, NoPatterns,
-    ParamsNotAllowed,
+    NakedFunctionsAsmBlock, NakedFunctionsMustNakedAsm, NoPatterns, ParamsNotAllowed,
 };
 
-pub(crate) fn provide(providers: &mut Providers) {
-    *providers = Providers { check_mod_naked_functions, ..*providers };
-}
-
-fn check_mod_naked_functions(tcx: TyCtxt<'_>, module_def_id: LocalModDefId) {
-    let items = tcx.hir_module_items(module_def_id);
-    for def_id in items.definitions() {
-        if !matches!(tcx.def_kind(def_id), DefKind::Fn | DefKind::AssocFn) {
-            continue;
-        }
-
-        let body = match tcx.hir_node_by_def_id(def_id) {
-            hir::Node::Item(hir::Item {
-                kind: hir::ItemKind::Fn { body: body_id, .. }, ..
-            })
-            | hir::Node::TraitItem(hir::TraitItem {
-                kind: hir::TraitItemKind::Fn(_, hir::TraitFn::Provided(body_id)),
-                ..
-            })
-            | hir::Node::ImplItem(hir::ImplItem {
-                kind: hir::ImplItemKind::Fn(_, body_id), ..
-            }) => tcx.hir_body(*body_id),
-            _ => continue,
-        };
-
-        if tcx.has_attr(def_id, sym::naked) {
-            check_no_patterns(tcx, body.params);
-            check_no_parameters_use(tcx, body);
-            check_asm(tcx, def_id, body);
-        } else {
-            // `naked_asm!` is not allowed outside of functions marked as `#[naked]`
-            let mut visitor = CheckNakedAsmInNakedFn { tcx };
-            visitor.visit_body(body);
-        }
-    }
+/// Naked fns can only have trivial binding patterns in arguments,
+/// may not actually use those arguments, and the body must consist of just
+/// a single asm statement.
+pub(crate) fn typeck_naked_fn<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    def_id: LocalDefId,
+    body: &'tcx hir::Body<'tcx>,
+) {
+    debug_assert!(tcx.has_attr(def_id, sym::naked));
+    check_no_patterns(tcx, body.params);
+    check_no_parameters_use(tcx, body);
+    check_asm(tcx, def_id, body);
 }
 
 /// Checks that parameters don't use patterns. Mirrors the checks for function declarations.
@@ -231,25 +204,3 @@ impl<'tcx> Visitor<'tcx> for CheckInlineAssembly {
         self.check_expr(expr, expr.span);
     }
 }
-
-struct CheckNakedAsmInNakedFn<'tcx> {
-    tcx: TyCtxt<'tcx>,
-}
-
-impl<'tcx> Visitor<'tcx> for CheckNakedAsmInNakedFn<'tcx> {
-    type NestedFilter = OnlyBodies;
-
-    fn maybe_tcx(&mut self) -> Self::MaybeTyCtxt {
-        self.tcx
-    }
-
-    fn visit_expr(&mut self, expr: &'tcx hir::Expr<'tcx>) {
-        if let ExprKind::InlineAsm(inline_asm) = expr.kind {
-            if let rustc_ast::AsmMacro::NakedAsm = inline_asm.asm_macro {
-                self.tcx.dcx().emit_err(NakedAsmOutsideNakedFn { span: expr.span });
-            }
-        }
-
-        hir::intravisit::walk_expr(self, expr);
-    }
-}
diff --git a/compiler/rustc_hir_typeck/src/pat.rs b/compiler/rustc_hir_typeck/src/pat.rs
index 17d48184dd9..432eeae8016 100644
--- a/compiler/rustc_hir_typeck/src/pat.rs
+++ b/compiler/rustc_hir_typeck/src/pat.rs
@@ -796,7 +796,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 if *negated {
                     self.register_bound(
                         ty,
-                        self.tcx.require_lang_item(LangItem::Neg, Some(lt.span)),
+                        self.tcx.require_lang_item(LangItem::Neg, lt.span),
                         ObligationCause::dummy_with_span(lt.span),
                     );
                 }
@@ -2553,13 +2553,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let tcx = self.tcx;
         self.register_bound(
             source_ty,
-            tcx.require_lang_item(hir::LangItem::DerefPure, Some(span)),
+            tcx.require_lang_item(hir::LangItem::DerefPure, span),
             self.misc(span),
         );
         // The expected type for the deref pat's inner pattern is `<expected as Deref>::Target`.
         let target_ty = Ty::new_projection(
             tcx,
-            tcx.require_lang_item(hir::LangItem::DerefTarget, Some(span)),
+            tcx.require_lang_item(hir::LangItem::DerefTarget, span),
             [source_ty],
         );
         let target_ty = self.normalize(span, target_ty);
@@ -2580,7 +2580,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             for mutably_derefed_ty in derefed_tys {
                 self.register_bound(
                     mutably_derefed_ty,
-                    self.tcx.require_lang_item(hir::LangItem::DerefMut, Some(span)),
+                    self.tcx.require_lang_item(hir::LangItem::DerefMut, span),
                     self.misc(span),
                 );
             }
diff --git a/compiler/rustc_hir_typeck/src/upvar.rs b/compiler/rustc_hir_typeck/src/upvar.rs
index 8ab71e5220b..5b5253c7e7e 100644
--- a/compiler/rustc_hir_typeck/src/upvar.rs
+++ b/compiler/rustc_hir_typeck/src/upvar.rs
@@ -902,7 +902,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 fn is_field<'a>(p: &&Projection<'a>) -> bool {
                     match p.kind {
                         ProjectionKind::Field(_, _) => true,
-                        ProjectionKind::Deref | ProjectionKind::OpaqueCast => false,
+                        ProjectionKind::Deref
+                        | ProjectionKind::OpaqueCast
+                        | ProjectionKind::UnwrapUnsafeBinder => false,
                         p @ (ProjectionKind::Subslice | ProjectionKind::Index) => {
                             bug!("ProjectionKind {:?} was unexpected", p)
                         }
@@ -1558,7 +1560,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         };
 
         let is_drop_defined_for_ty = |ty: Ty<'tcx>| {
-            let drop_trait = self.tcx.require_lang_item(hir::LangItem::Drop, Some(closure_span));
+            let drop_trait = self.tcx.require_lang_item(hir::LangItem::Drop, closure_span);
             self.infcx
                 .type_implements_trait(drop_trait, [ty], self.tcx.param_env(closure_def_id))
                 .must_apply_modulo_regions()
@@ -2197,7 +2199,8 @@ fn restrict_capture_precision(
             }
             ProjectionKind::Deref => {}
             ProjectionKind::OpaqueCast => {}
-            ProjectionKind::Field(..) => {} // ignore
+            ProjectionKind::Field(..) => {}
+            ProjectionKind::UnwrapUnsafeBinder => {}
         }
     }
 
@@ -2268,6 +2271,7 @@ fn construct_place_string<'tcx>(tcx: TyCtxt<'_>, place: &Place<'tcx>) -> String
             ProjectionKind::Index => String::from("Index"),
             ProjectionKind::Subslice => String::from("Subslice"),
             ProjectionKind::OpaqueCast => String::from("OpaqueCast"),
+            ProjectionKind::UnwrapUnsafeBinder => String::from("UnwrapUnsafeBinder"),
         };
         if i != 0 {
             projections_str.push(',');
diff --git a/compiler/rustc_hir_typeck/src/writeback.rs b/compiler/rustc_hir_typeck/src/writeback.rs
index 9be041f75d7..b2497cb0de1 100644
--- a/compiler/rustc_hir_typeck/src/writeback.rs
+++ b/compiler/rustc_hir_typeck/src/writeback.rs
@@ -9,17 +9,21 @@
 //! which creates a new `TypeckResults` which doesn't contain any inference variables.
 
 use std::mem;
+use std::ops::ControlFlow;
 
+use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
 use rustc_data_structures::unord::ExtendUnord;
-use rustc_errors::ErrorGuaranteed;
+use rustc_errors::{E0720, ErrorGuaranteed};
+use rustc_hir::def_id::LocalDefId;
 use rustc_hir::intravisit::{self, InferKind, Visitor};
 use rustc_hir::{self as hir, AmbigArg, HirId};
 use rustc_infer::traits::solve::Goal;
 use rustc_middle::traits::ObligationCause;
 use rustc_middle::ty::adjustment::{Adjust, Adjustment, PointerCoercion};
 use rustc_middle::ty::{
-    self, DefiningScopeKind, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeSuperFoldable,
-    TypeVisitableExt, fold_regions,
+    self, DefiningScopeKind, OpaqueHiddenType, Ty, TyCtxt, TypeFoldable, TypeFolder,
+    TypeSuperFoldable, TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor,
+    fold_regions,
 };
 use rustc_span::{Span, sym};
 use rustc_trait_selection::error_reporting::infer::need_type_info::TypeAnnotationNeeded;
@@ -595,6 +599,35 @@ impl<'cx, 'tcx> WritebackCx<'cx, 'tcx> {
                 entry.span = prev.span.substitute_dummy(hidden_type.span);
             }
         }
+
+        let recursive_opaques: Vec<_> = self
+            .typeck_results
+            .concrete_opaque_types
+            .iter()
+            .filter(|&(&def_id, hidden_ty)| {
+                hidden_ty
+                    .ty
+                    .visit_with(&mut HasRecursiveOpaque {
+                        def_id,
+                        seen: Default::default(),
+                        opaques: &self.typeck_results.concrete_opaque_types,
+                        tcx,
+                    })
+                    .is_break()
+            })
+            .map(|(def_id, hidden_ty)| (*def_id, hidden_ty.span))
+            .collect();
+        for (def_id, span) in recursive_opaques {
+            let guar = self
+                .fcx
+                .dcx()
+                .struct_span_err(span, "cannot resolve opaque type")
+                .with_code(E0720)
+                .emit();
+            self.typeck_results
+                .concrete_opaque_types
+                .insert(def_id, OpaqueHiddenType { span, ty: Ty::new_error(tcx, guar) });
+        }
     }
 
     fn visit_field_id(&mut self, hir_id: HirId) {
@@ -959,3 +992,34 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for EagerlyNormalizeConsts<'tcx> {
         self.tcx.try_normalize_erasing_regions(self.typing_env, ct).unwrap_or(ct)
     }
 }
+
+struct HasRecursiveOpaque<'a, 'tcx> {
+    def_id: LocalDefId,
+    seen: FxHashSet<LocalDefId>,
+    opaques: &'a FxIndexMap<LocalDefId, ty::OpaqueHiddenType<'tcx>>,
+    tcx: TyCtxt<'tcx>,
+}
+
+impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for HasRecursiveOpaque<'_, 'tcx> {
+    type Result = ControlFlow<()>;
+
+    fn visit_ty(&mut self, t: Ty<'tcx>) -> Self::Result {
+        if let ty::Alias(ty::Opaque, alias_ty) = *t.kind()
+            && let Some(def_id) = alias_ty.def_id.as_local()
+        {
+            if self.def_id == def_id {
+                return ControlFlow::Break(());
+            }
+
+            if self.seen.insert(def_id)
+                && let Some(hidden_ty) = self.opaques.get(&def_id)
+            {
+                ty::EarlyBinder::bind(hidden_ty.ty)
+                    .instantiate(self.tcx, alias_ty.args)
+                    .visit_with(self)?;
+            }
+        }
+
+        t.super_visit_with(self)
+    }
+}
diff --git a/compiler/rustc_index/src/bit_set.rs b/compiler/rustc_index/src/bit_set.rs
index 07934389158..a4885aabe1f 100644
--- a/compiler/rustc_index/src/bit_set.rs
+++ b/compiler/rustc_index/src/bit_set.rs
@@ -234,6 +234,32 @@ impl<T: Idx> DenseBitSet<T> {
         self.clear_excess_bits();
     }
 
+    /// Checks whether any bit in the given range is a 1.
+    #[inline]
+    pub fn contains_any(&self, elems: impl RangeBounds<T>) -> bool {
+        let Some((start, end)) = inclusive_start_end(elems, self.domain_size) else {
+            return false;
+        };
+        let (start_word_index, start_mask) = word_index_and_mask(start);
+        let (end_word_index, end_mask) = word_index_and_mask(end);
+
+        if start_word_index == end_word_index {
+            self.words[start_word_index] & (end_mask | (end_mask - start_mask)) != 0
+        } else {
+            if self.words[start_word_index] & !(start_mask - 1) != 0 {
+                return true;
+            }
+
+            let remaining = start_word_index + 1..end_word_index;
+            if remaining.start <= remaining.end {
+                self.words[remaining].iter().any(|&w| w != 0)
+                    || self.words[end_word_index] & (end_mask | (end_mask - 1)) != 0
+            } else {
+                false
+            }
+        }
+    }
+
     /// Returns `true` if the set has changed.
     #[inline]
     pub fn remove(&mut self, elem: T) -> bool {
diff --git a/compiler/rustc_index/src/bit_set/tests.rs b/compiler/rustc_index/src/bit_set/tests.rs
index 323a66ddc6f..9ce4cf4293f 100644
--- a/compiler/rustc_index/src/bit_set/tests.rs
+++ b/compiler/rustc_index/src/bit_set/tests.rs
@@ -692,6 +692,25 @@ fn dense_last_set_before() {
     }
 }
 
+#[test]
+fn dense_contains_any() {
+    let mut set: DenseBitSet<usize> = DenseBitSet::new_empty(300);
+    assert!(!set.contains_any(0..300));
+    set.insert_range(10..20);
+    set.insert_range(60..70);
+    set.insert_range(150..=250);
+
+    assert!(set.contains_any(0..30));
+    assert!(set.contains_any(5..100));
+    assert!(set.contains_any(250..255));
+
+    assert!(!set.contains_any(20..59));
+    assert!(!set.contains_any(256..290));
+
+    set.insert(22);
+    assert!(set.contains_any(20..59));
+}
+
 #[bench]
 fn bench_insert(b: &mut Bencher) {
     let mut bs = DenseBitSet::new_filled(99999usize);
diff --git a/compiler/rustc_infer/src/infer/at.rs b/compiler/rustc_infer/src/infer/at.rs
index 2cd67cc4da2..5fe795bd23a 100644
--- a/compiler/rustc_infer/src/infer/at.rs
+++ b/compiler/rustc_infer/src/infer/at.rs
@@ -347,7 +347,7 @@ impl<'tcx> ToTrace<'tcx> for ty::GenericArg<'tcx> {
     fn to_trace(cause: &ObligationCause<'tcx>, a: Self, b: Self) -> TypeTrace<'tcx> {
         TypeTrace {
             cause: cause.clone(),
-            values: match (a.unpack(), b.unpack()) {
+            values: match (a.kind(), b.kind()) {
                 (GenericArgKind::Lifetime(a), GenericArgKind::Lifetime(b)) => {
                     ValuePairs::Regions(ExpectedFound::new(a, b))
                 }
diff --git a/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs b/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs
index a1a0926cd81..060447ba720 100644
--- a/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs
+++ b/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs
@@ -17,8 +17,7 @@ use tracing::debug;
 
 use crate::infer::InferCtxt;
 use crate::infer::canonical::{
-    Canonical, CanonicalQueryInput, CanonicalTyVarKind, CanonicalVarInfo, CanonicalVarKind,
-    OriginalQueryValues,
+    Canonical, CanonicalQueryInput, CanonicalTyVarKind, CanonicalVarKind, OriginalQueryValues,
 };
 
 impl<'tcx> InferCtxt<'tcx> {
@@ -174,10 +173,8 @@ impl CanonicalizeMode for CanonicalizeQueryResponse {
         match r.kind() {
             ty::ReLateParam(_) | ty::ReErased | ty::ReStatic | ty::ReEarlyParam(..) => r,
 
-            ty::RePlaceholder(placeholder) => canonicalizer.canonical_var_for_region(
-                CanonicalVarInfo { kind: CanonicalVarKind::PlaceholderRegion(placeholder) },
-                r,
-            ),
+            ty::RePlaceholder(placeholder) => canonicalizer
+                .canonical_var_for_region(CanonicalVarKind::PlaceholderRegion(placeholder), r),
 
             ty::ReVar(vid) => {
                 let universe = infcx
@@ -186,10 +183,7 @@ impl CanonicalizeMode for CanonicalizeQueryResponse {
                     .unwrap_region_constraints()
                     .probe_value(vid)
                     .unwrap_err();
-                canonicalizer.canonical_var_for_region(
-                    CanonicalVarInfo { kind: CanonicalVarKind::Region(universe) },
-                    r,
-                )
+                canonicalizer.canonical_var_for_region(CanonicalVarKind::Region(universe), r)
             }
 
             _ => {
@@ -294,7 +288,7 @@ struct Canonicalizer<'cx, 'tcx> {
     /// Set to `None` to disable the resolution of inference variables.
     infcx: Option<&'cx InferCtxt<'tcx>>,
     tcx: TyCtxt<'tcx>,
-    variables: SmallVec<[CanonicalVarInfo<'tcx>; 8]>,
+    variables: SmallVec<[CanonicalVarKind<'tcx>; 8]>,
     query_state: &'cx mut OriginalQueryValues<'tcx>,
     // Note that indices is only used once `var_values` is big enough to be
     // heap-allocated.
@@ -368,9 +362,7 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
                             ui = ty::UniverseIndex::ROOT;
                         }
                         self.canonicalize_ty_var(
-                            CanonicalVarInfo {
-                                kind: CanonicalVarKind::Ty(CanonicalTyVarKind::General(ui)),
-                            },
+                            CanonicalVarKind::Ty(CanonicalTyVarKind::General(ui)),
                             t,
                         )
                     }
@@ -382,10 +374,7 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
                 if nt != t {
                     return self.fold_ty(nt);
                 } else {
-                    self.canonicalize_ty_var(
-                        CanonicalVarInfo { kind: CanonicalVarKind::Ty(CanonicalTyVarKind::Int) },
-                        t,
-                    )
+                    self.canonicalize_ty_var(CanonicalVarKind::Ty(CanonicalTyVarKind::Int), t)
                 }
             }
             ty::Infer(ty::FloatVar(vid)) => {
@@ -393,10 +382,7 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
                 if nt != t {
                     return self.fold_ty(nt);
                 } else {
-                    self.canonicalize_ty_var(
-                        CanonicalVarInfo { kind: CanonicalVarKind::Ty(CanonicalTyVarKind::Float) },
-                        t,
-                    )
+                    self.canonicalize_ty_var(CanonicalVarKind::Ty(CanonicalTyVarKind::Float), t)
                 }
             }
 
@@ -408,10 +394,7 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
                 if !self.canonicalize_mode.preserve_universes() {
                     placeholder.universe = ty::UniverseIndex::ROOT;
                 }
-                self.canonicalize_ty_var(
-                    CanonicalVarInfo { kind: CanonicalVarKind::PlaceholderTy(placeholder) },
-                    t,
-                )
+                self.canonicalize_ty_var(CanonicalVarKind::PlaceholderTy(placeholder), t)
             }
 
             ty::Bound(debruijn, _) => {
@@ -483,10 +466,7 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
                             // FIXME: perf problem described in #55921.
                             ui = ty::UniverseIndex::ROOT;
                         }
-                        return self.canonicalize_const_var(
-                            CanonicalVarInfo { kind: CanonicalVarKind::Const(ui) },
-                            ct,
-                        );
+                        return self.canonicalize_const_var(CanonicalVarKind::Const(ui), ct);
                     }
                 }
             }
@@ -501,10 +481,8 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
                 }
             }
             ty::ConstKind::Placeholder(placeholder) => {
-                return self.canonicalize_const_var(
-                    CanonicalVarInfo { kind: CanonicalVarKind::PlaceholderConst(placeholder) },
-                    ct,
-                );
+                return self
+                    .canonicalize_const_var(CanonicalVarKind::PlaceholderConst(placeholder), ct);
             }
             _ => {}
         }
@@ -515,6 +493,14 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
             ct
         }
     }
+
+    fn fold_predicate(&mut self, p: ty::Predicate<'tcx>) -> ty::Predicate<'tcx> {
+        if p.flags().intersects(self.needs_canonical_flags) { p.super_fold_with(self) } else { p }
+    }
+
+    fn fold_clauses(&mut self, c: ty::Clauses<'tcx>) -> ty::Clauses<'tcx> {
+        if c.flags().intersects(self.needs_canonical_flags) { c.super_fold_with(self) } else { c }
+    }
 }
 
 impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
@@ -595,7 +581,7 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
         debug_assert!(!out_value.has_infer() && !out_value.has_placeholders());
 
         let canonical_variables =
-            tcx.mk_canonical_var_infos(&canonicalizer.universe_canonicalized_variables());
+            tcx.mk_canonical_var_kinds(&canonicalizer.universe_canonicalized_variables());
 
         let max_universe = canonical_variables
             .iter()
@@ -610,18 +596,22 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
     /// or returns an existing variable if `kind` has already been
     /// seen. `kind` is expected to be an unbound variable (or
     /// potentially a free region).
-    fn canonical_var(&mut self, info: CanonicalVarInfo<'tcx>, kind: GenericArg<'tcx>) -> BoundVar {
+    fn canonical_var(
+        &mut self,
+        var_kind: CanonicalVarKind<'tcx>,
+        value: GenericArg<'tcx>,
+    ) -> BoundVar {
         let Canonicalizer { variables, query_state, indices, .. } = self;
 
         let var_values = &mut query_state.var_values;
 
-        let universe = info.universe();
+        let universe = var_kind.universe();
         if universe != ty::UniverseIndex::ROOT {
             assert!(self.canonicalize_mode.preserve_universes());
 
             // Insert universe into the universe map. To preserve the order of the
             // universes in the value being canonicalized, we don't update the
-            // universe in `info` until we have finished canonicalizing.
+            // universe in `var_kind` until we have finished canonicalizing.
             match query_state.universe_map.binary_search(&universe) {
                 Err(idx) => query_state.universe_map.insert(idx, universe),
                 Ok(_) => {}
@@ -636,14 +626,14 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
         if !var_values.spilled() {
             // `var_values` is stack-allocated. `indices` isn't used yet. Do a
             // direct linear search of `var_values`.
-            if let Some(idx) = var_values.iter().position(|&k| k == kind) {
+            if let Some(idx) = var_values.iter().position(|&v| v == value) {
                 // `kind` is already present in `var_values`.
                 BoundVar::new(idx)
             } else {
                 // `kind` isn't present in `var_values`. Append it. Likewise
-                // for `info` and `variables`.
-                variables.push(info);
-                var_values.push(kind);
+                // for `var_kind` and `variables`.
+                variables.push(var_kind);
+                var_values.push(value);
                 assert_eq!(variables.len(), var_values.len());
 
                 // If `var_values` has become big enough to be heap-allocated,
@@ -653,7 +643,7 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
                     *indices = var_values
                         .iter()
                         .enumerate()
-                        .map(|(i, &kind)| (kind, BoundVar::new(i)))
+                        .map(|(i, &value)| (value, BoundVar::new(i)))
                         .collect();
                 }
                 // The cv is the index of the appended element.
@@ -661,9 +651,9 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
             }
         } else {
             // `var_values` is large. Do a hashmap search via `indices`.
-            *indices.entry(kind).or_insert_with(|| {
-                variables.push(info);
-                var_values.push(kind);
+            *indices.entry(value).or_insert_with(|| {
+                variables.push(var_kind);
+                var_values.push(value);
                 assert_eq!(variables.len(), var_values.len());
                 BoundVar::new(variables.len() - 1)
             })
@@ -673,7 +663,7 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
     /// Replaces the universe indexes used in `var_values` with their index in
     /// `query_state.universe_map`. This minimizes the maximum universe used in
     /// the canonicalized value.
-    fn universe_canonicalized_variables(self) -> SmallVec<[CanonicalVarInfo<'tcx>; 8]> {
+    fn universe_canonicalized_variables(self) -> SmallVec<[CanonicalVarKind<'tcx>; 8]> {
         if self.query_state.universe_map.len() == 1 {
             return self.variables;
         }
@@ -688,37 +678,33 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
 
         self.variables
             .iter()
-            .map(|v| CanonicalVarInfo {
-                kind: match v.kind {
-                    CanonicalVarKind::Ty(CanonicalTyVarKind::Int | CanonicalTyVarKind::Float) => {
-                        return *v;
-                    }
-                    CanonicalVarKind::Ty(CanonicalTyVarKind::General(u)) => {
-                        CanonicalVarKind::Ty(CanonicalTyVarKind::General(reverse_universe_map[&u]))
-                    }
-                    CanonicalVarKind::Region(u) => {
-                        CanonicalVarKind::Region(reverse_universe_map[&u])
-                    }
-                    CanonicalVarKind::Const(u) => CanonicalVarKind::Const(reverse_universe_map[&u]),
-                    CanonicalVarKind::PlaceholderTy(placeholder) => {
-                        CanonicalVarKind::PlaceholderTy(ty::Placeholder {
-                            universe: reverse_universe_map[&placeholder.universe],
-                            ..placeholder
-                        })
-                    }
-                    CanonicalVarKind::PlaceholderRegion(placeholder) => {
-                        CanonicalVarKind::PlaceholderRegion(ty::Placeholder {
-                            universe: reverse_universe_map[&placeholder.universe],
-                            ..placeholder
-                        })
-                    }
-                    CanonicalVarKind::PlaceholderConst(placeholder) => {
-                        CanonicalVarKind::PlaceholderConst(ty::Placeholder {
-                            universe: reverse_universe_map[&placeholder.universe],
-                            ..placeholder
-                        })
-                    }
-                },
+            .map(|&kind| match kind {
+                CanonicalVarKind::Ty(CanonicalTyVarKind::Int | CanonicalTyVarKind::Float) => {
+                    return kind;
+                }
+                CanonicalVarKind::Ty(CanonicalTyVarKind::General(u)) => {
+                    CanonicalVarKind::Ty(CanonicalTyVarKind::General(reverse_universe_map[&u]))
+                }
+                CanonicalVarKind::Region(u) => CanonicalVarKind::Region(reverse_universe_map[&u]),
+                CanonicalVarKind::Const(u) => CanonicalVarKind::Const(reverse_universe_map[&u]),
+                CanonicalVarKind::PlaceholderTy(placeholder) => {
+                    CanonicalVarKind::PlaceholderTy(ty::Placeholder {
+                        universe: reverse_universe_map[&placeholder.universe],
+                        ..placeholder
+                    })
+                }
+                CanonicalVarKind::PlaceholderRegion(placeholder) => {
+                    CanonicalVarKind::PlaceholderRegion(ty::Placeholder {
+                        universe: reverse_universe_map[&placeholder.universe],
+                        ..placeholder
+                    })
+                }
+                CanonicalVarKind::PlaceholderConst(placeholder) => {
+                    CanonicalVarKind::PlaceholderConst(ty::Placeholder {
+                        universe: reverse_universe_map[&placeholder.universe],
+                        ..placeholder
+                    })
+                }
             })
             .collect()
     }
@@ -740,20 +726,17 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
         &mut self,
         r: ty::Region<'tcx>,
     ) -> ty::Region<'tcx> {
-        self.canonical_var_for_region(
-            CanonicalVarInfo { kind: CanonicalVarKind::Region(ty::UniverseIndex::ROOT) },
-            r,
-        )
+        self.canonical_var_for_region(CanonicalVarKind::Region(ty::UniverseIndex::ROOT), r)
     }
 
     /// Creates a canonical variable (with the given `info`)
     /// representing the region `r`; return a region referencing it.
     fn canonical_var_for_region(
         &mut self,
-        info: CanonicalVarInfo<'tcx>,
+        var_kind: CanonicalVarKind<'tcx>,
         r: ty::Region<'tcx>,
     ) -> ty::Region<'tcx> {
-        let var = self.canonical_var(info, r.into());
+        let var = self.canonical_var(var_kind, r.into());
         let br = ty::BoundRegion { var, kind: ty::BoundRegionKind::Anon };
         ty::Region::new_bound(self.cx(), self.binder_index, br)
     }
@@ -762,9 +745,13 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
     /// if `ty_var` is bound to anything; if so, canonicalize
     /// *that*. Otherwise, create a new canonical variable for
     /// `ty_var`.
-    fn canonicalize_ty_var(&mut self, info: CanonicalVarInfo<'tcx>, ty_var: Ty<'tcx>) -> Ty<'tcx> {
+    fn canonicalize_ty_var(
+        &mut self,
+        var_kind: CanonicalVarKind<'tcx>,
+        ty_var: Ty<'tcx>,
+    ) -> Ty<'tcx> {
         debug_assert!(!self.infcx.is_some_and(|infcx| ty_var != infcx.shallow_resolve(ty_var)));
-        let var = self.canonical_var(info, ty_var.into());
+        let var = self.canonical_var(var_kind, ty_var.into());
         Ty::new_bound(self.tcx, self.binder_index, var.into())
     }
 
@@ -774,13 +761,13 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
     /// `const_var`.
     fn canonicalize_const_var(
         &mut self,
-        info: CanonicalVarInfo<'tcx>,
-        const_var: ty::Const<'tcx>,
+        var_kind: CanonicalVarKind<'tcx>,
+        ct_var: ty::Const<'tcx>,
     ) -> ty::Const<'tcx> {
         debug_assert!(
-            !self.infcx.is_some_and(|infcx| const_var != infcx.shallow_resolve_const(const_var))
+            !self.infcx.is_some_and(|infcx| ct_var != infcx.shallow_resolve_const(ct_var))
         );
-        let var = self.canonical_var(info, const_var.into());
+        let var = self.canonical_var(var_kind, ct_var.into());
         ty::Const::new_bound(self.tcx, self.binder_index, var)
     }
 }
diff --git a/compiler/rustc_infer/src/infer/canonical/instantiate.rs b/compiler/rustc_infer/src/infer/canonical/instantiate.rs
index f5ee5702d09..67f13192b52 100644
--- a/compiler/rustc_infer/src/infer/canonical/instantiate.rs
+++ b/compiler/rustc_infer/src/infer/canonical/instantiate.rs
@@ -61,15 +61,15 @@ where
         value
     } else {
         let delegate = FnMutDelegate {
-            regions: &mut |br: ty::BoundRegion| match var_values[br.var].unpack() {
+            regions: &mut |br: ty::BoundRegion| match var_values[br.var].kind() {
                 GenericArgKind::Lifetime(l) => l,
                 r => bug!("{:?} is a region but value is {:?}", br, r),
             },
-            types: &mut |bound_ty: ty::BoundTy| match var_values[bound_ty.var].unpack() {
+            types: &mut |bound_ty: ty::BoundTy| match var_values[bound_ty.var].kind() {
                 GenericArgKind::Type(ty) => ty,
                 r => bug!("{:?} is a type but value is {:?}", bound_ty, r),
             },
-            consts: &mut |bound_ct: ty::BoundVar| match var_values[bound_ct].unpack() {
+            consts: &mut |bound_ct: ty::BoundVar| match var_values[bound_ct].kind() {
                 GenericArgKind::Const(ct) => ct,
                 c => bug!("{:?} is a const but value is {:?}", bound_ct, c),
             },
diff --git a/compiler/rustc_infer/src/infer/canonical/mod.rs b/compiler/rustc_infer/src/infer/canonical/mod.rs
index 3be07dbe208..5dffedc7099 100644
--- a/compiler/rustc_infer/src/infer/canonical/mod.rs
+++ b/compiler/rustc_infer/src/infer/canonical/mod.rs
@@ -81,14 +81,14 @@ impl<'tcx> InferCtxt<'tcx> {
     fn instantiate_canonical_vars(
         &self,
         span: Span,
-        variables: &List<CanonicalVarInfo<'tcx>>,
+        variables: &List<CanonicalVarKind<'tcx>>,
         universe_map: impl Fn(ty::UniverseIndex) -> ty::UniverseIndex,
     ) -> CanonicalVarValues<'tcx> {
         CanonicalVarValues {
             var_values: self.tcx.mk_args_from_iter(
                 variables
                     .iter()
-                    .map(|info| self.instantiate_canonical_var(span, info, &universe_map)),
+                    .map(|kind| self.instantiate_canonical_var(span, kind, &universe_map)),
             ),
         }
     }
@@ -104,10 +104,10 @@ impl<'tcx> InferCtxt<'tcx> {
     pub fn instantiate_canonical_var(
         &self,
         span: Span,
-        cv_info: CanonicalVarInfo<'tcx>,
+        kind: CanonicalVarKind<'tcx>,
         universe_map: impl Fn(ty::UniverseIndex) -> ty::UniverseIndex,
     ) -> GenericArg<'tcx> {
-        match cv_info.kind {
+        match kind {
             CanonicalVarKind::Ty(ty_kind) => {
                 let ty = match ty_kind {
                     CanonicalTyVarKind::General(ui) => {
diff --git a/compiler/rustc_infer/src/infer/canonical/query_response.rs b/compiler/rustc_infer/src/infer/canonical/query_response.rs
index 1ae864c454f..008ef690008 100644
--- a/compiler/rustc_infer/src/infer/canonical/query_response.rs
+++ b/compiler/rustc_infer/src/infer/canonical/query_response.rs
@@ -12,23 +12,20 @@ use std::iter;
 
 use rustc_index::{Idx, IndexVec};
 use rustc_middle::arena::ArenaAllocatable;
+use rustc_middle::bug;
 use rustc_middle::mir::ConstraintCategory;
 use rustc_middle::ty::{self, BoundVar, GenericArg, GenericArgKind, Ty, TyCtxt, TypeFoldable};
-use rustc_middle::{bug, span_bug};
 use tracing::{debug, instrument};
 
 use crate::infer::canonical::instantiate::{CanonicalExt, instantiate_value};
 use crate::infer::canonical::{
     Canonical, CanonicalQueryResponse, CanonicalVarValues, Certainty, OriginalQueryValues,
-    QueryOutlivesConstraint, QueryRegionConstraints, QueryResponse,
+    QueryRegionConstraints, QueryResponse,
 };
 use crate::infer::region_constraints::{Constraint, RegionConstraintData};
-use crate::infer::{DefineOpaqueTypes, InferCtxt, InferOk, InferResult};
+use crate::infer::{DefineOpaqueTypes, InferCtxt, InferOk, InferResult, SubregionOrigin};
 use crate::traits::query::NoSolution;
-use crate::traits::{
-    Obligation, ObligationCause, PredicateObligation, PredicateObligations, ScrubbedTraitError,
-    TraitEngine,
-};
+use crate::traits::{ObligationCause, PredicateObligations, ScrubbedTraitError, TraitEngine};
 
 impl<'tcx> InferCtxt<'tcx> {
     /// This method is meant to be invoked as the final step of a canonical query
@@ -169,15 +166,13 @@ impl<'tcx> InferCtxt<'tcx> {
     where
         R: Debug + TypeFoldable<TyCtxt<'tcx>>,
     {
-        let InferOk { value: result_args, mut obligations } =
+        let InferOk { value: result_args, obligations } =
             self.query_response_instantiation(cause, param_env, original_values, query_response)?;
 
-        obligations.extend(self.query_outlives_constraints_into_obligations(
-            cause,
-            param_env,
-            &query_response.value.region_constraints.outlives,
-            &result_args,
-        ));
+        for (predicate, _category) in &query_response.value.region_constraints.outlives {
+            let predicate = instantiate_value(self.tcx, &result_args, *predicate);
+            self.register_outlives_constraint(predicate, cause);
+        }
 
         let user_result: R =
             query_response.instantiate_projected(self.tcx, &result_args, |q_r| q_r.value.clone());
@@ -250,7 +245,7 @@ impl<'tcx> InferCtxt<'tcx> {
             let result_value = query_response.instantiate_projected(self.tcx, &result_args, |v| {
                 v.var_values[BoundVar::new(index)]
             });
-            match (original_value.unpack(), result_value.unpack()) {
+            match (original_value.kind(), result_value.kind()) {
                 (GenericArgKind::Lifetime(re1), GenericArgKind::Lifetime(re2))
                     if re1.is_erased() && re2.is_erased() =>
                 {
@@ -407,7 +402,7 @@ impl<'tcx> InferCtxt<'tcx> {
         // [(?A, Vec<?0>), ('static, '?1), (?B, ?0)]
         for (original_value, result_value) in iter::zip(&original_values.var_values, result_values)
         {
-            match result_value.unpack() {
+            match result_value.kind() {
                 GenericArgKind::Type(result_value) => {
                     // e.g., here `result_value` might be `?0` in the example above...
                     if let ty::Bound(debruijn, b) = *result_value.kind() {
@@ -445,17 +440,17 @@ impl<'tcx> InferCtxt<'tcx> {
         // a fresh inference variable.
         let result_args = CanonicalVarValues {
             var_values: self.tcx.mk_args_from_iter(
-                query_response.variables.iter().enumerate().map(|(index, info)| {
-                    if info.universe() != ty::UniverseIndex::ROOT {
+                query_response.variables.iter().enumerate().map(|(index, var_kind)| {
+                    if var_kind.universe() != ty::UniverseIndex::ROOT {
                         // A variable from inside a binder of the query. While ideally these shouldn't
                         // exist at all, we have to deal with them for now.
-                        self.instantiate_canonical_var(cause.span, info, |u| {
+                        self.instantiate_canonical_var(cause.span, var_kind, |u| {
                             universe_map[u.as_usize()]
                         })
-                    } else if info.is_existential() {
+                    } else if var_kind.is_existential() {
                         match opt_values[BoundVar::new(index)] {
                             Some(k) => k,
-                            None => self.instantiate_canonical_var(cause.span, info, |u| {
+                            None => self.instantiate_canonical_var(cause.span, var_kind, |u| {
                                 universe_map[u.as_usize()]
                             }),
                         }
@@ -525,47 +520,6 @@ impl<'tcx> InferCtxt<'tcx> {
         self.unify_canonical_vars(cause, param_env, original_values, instantiated_query_response)
     }
 
-    /// Converts the region constraints resulting from a query into an
-    /// iterator of obligations.
-    fn query_outlives_constraints_into_obligations(
-        &self,
-        cause: &ObligationCause<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
-        uninstantiated_region_constraints: &[QueryOutlivesConstraint<'tcx>],
-        result_args: &CanonicalVarValues<'tcx>,
-    ) -> impl Iterator<Item = PredicateObligation<'tcx>> {
-        uninstantiated_region_constraints.iter().map(move |&constraint| {
-            let predicate = instantiate_value(self.tcx, result_args, constraint);
-            self.query_outlives_constraint_to_obligation(predicate, cause.clone(), param_env)
-        })
-    }
-
-    pub fn query_outlives_constraint_to_obligation(
-        &self,
-        (predicate, _): QueryOutlivesConstraint<'tcx>,
-        cause: ObligationCause<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
-    ) -> Obligation<'tcx, ty::Predicate<'tcx>> {
-        let ty::OutlivesPredicate(k1, r2) = predicate;
-
-        let atom = match k1.unpack() {
-            GenericArgKind::Lifetime(r1) => ty::PredicateKind::Clause(
-                ty::ClauseKind::RegionOutlives(ty::OutlivesPredicate(r1, r2)),
-            ),
-            GenericArgKind::Type(t1) => ty::PredicateKind::Clause(ty::ClauseKind::TypeOutlives(
-                ty::OutlivesPredicate(t1, r2),
-            )),
-            GenericArgKind::Const(..) => {
-                // Consts cannot outlive one another, so we don't expect to
-                // encounter this branch.
-                span_bug!(cause.span, "unexpected const outlives {:?}", predicate);
-            }
-        };
-        let predicate = ty::Binder::dummy(atom);
-
-        Obligation::new(self.tcx, cause, param_env, predicate)
-    }
-
     /// Given two sets of values for the same set of canonical variables, unify them.
     /// The second set is produced lazily by supplying indices from the first set.
     fn unify_canonical_vars(
@@ -579,7 +533,7 @@ impl<'tcx> InferCtxt<'tcx> {
         for (index, value1) in variables1.var_values.iter().enumerate() {
             let value2 = variables2(BoundVar::new(index));
 
-            match (value1.unpack(), value2.unpack()) {
+            match (value1.kind(), value2.kind()) {
                 (GenericArgKind::Type(v1), GenericArgKind::Type(v2)) => {
                     obligations.extend(
                         self.at(cause, param_env)
@@ -593,10 +547,10 @@ impl<'tcx> InferCtxt<'tcx> {
                     // no action needed
                 }
                 (GenericArgKind::Lifetime(v1), GenericArgKind::Lifetime(v2)) => {
-                    obligations.extend(
-                        self.at(cause, param_env)
-                            .eq(DefineOpaqueTypes::Yes, v1, v2)?
-                            .into_obligations(),
+                    self.inner.borrow_mut().unwrap_region_constraints().make_eqregion(
+                        SubregionOrigin::RelateRegionParamBound(cause.span, None),
+                        v1,
+                        v2,
                     );
                 }
                 (GenericArgKind::Const(v1), GenericArgKind::Const(v2)) => {
diff --git a/compiler/rustc_infer/src/infer/context.rs b/compiler/rustc_infer/src/infer/context.rs
index 359b9da11ce..f7c702d321b 100644
--- a/compiler/rustc_infer/src/infer/context.rs
+++ b/compiler/rustc_infer/src/infer/context.rs
@@ -89,6 +89,57 @@ impl<'tcx> rustc_type_ir::InferCtxtLike for InferCtxt<'tcx> {
         self.inner.borrow_mut().unwrap_region_constraints().opportunistic_resolve_var(self.tcx, vid)
     }
 
+    fn is_changed_arg(&self, arg: ty::GenericArg<'tcx>) -> bool {
+        match arg.kind() {
+            ty::GenericArgKind::Lifetime(_) => {
+                // Lifetimes should not change affect trait selection.
+                false
+            }
+            ty::GenericArgKind::Type(ty) => {
+                if let ty::Infer(infer_ty) = *ty.kind() {
+                    match infer_ty {
+                        ty::InferTy::TyVar(vid) => {
+                            !self.probe_ty_var(vid).is_err_and(|_| self.root_var(vid) == vid)
+                        }
+                        ty::InferTy::IntVar(vid) => {
+                            let mut inner = self.inner.borrow_mut();
+                            !matches!(
+                                inner.int_unification_table().probe_value(vid),
+                                ty::IntVarValue::Unknown
+                                    if inner.int_unification_table().find(vid) == vid
+                            )
+                        }
+                        ty::InferTy::FloatVar(vid) => {
+                            let mut inner = self.inner.borrow_mut();
+                            !matches!(
+                                inner.float_unification_table().probe_value(vid),
+                                ty::FloatVarValue::Unknown
+                                    if inner.float_unification_table().find(vid) == vid
+                            )
+                        }
+                        ty::InferTy::FreshTy(_)
+                        | ty::InferTy::FreshIntTy(_)
+                        | ty::InferTy::FreshFloatTy(_) => true,
+                    }
+                } else {
+                    true
+                }
+            }
+            ty::GenericArgKind::Const(ct) => {
+                if let ty::ConstKind::Infer(infer_ct) = ct.kind() {
+                    match infer_ct {
+                        ty::InferConst::Var(vid) => !self
+                            .probe_const_var(vid)
+                            .is_err_and(|_| self.root_const_var(vid) == vid),
+                        ty::InferConst::Fresh(_) => true,
+                    }
+                } else {
+                    true
+                }
+            }
+        }
+    }
+
     fn next_region_infer(&self) -> ty::Region<'tcx> {
         self.next_region_var(RegionVariableOrigin::MiscVariable(DUMMY_SP))
     }
@@ -214,7 +265,7 @@ impl<'tcx> rustc_type_ir::InferCtxtLike for InferCtxt<'tcx> {
     }
 
     fn register_ty_outlives(&self, ty: Ty<'tcx>, r: ty::Region<'tcx>, span: Span) {
-        self.register_region_obligation_with_cause(ty, r, &ObligationCause::dummy_with_span(span));
+        self.register_type_outlives_constraint(ty, r, &ObligationCause::dummy_with_span(span));
     }
 
     type OpaqueTypeStorageEntries = OpaqueTypeStorageEntries;
diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs
index b408d76010d..e9b58eb959b 100644
--- a/compiler/rustc_infer/src/infer/mod.rs
+++ b/compiler/rustc_infer/src/infer/mod.rs
@@ -150,7 +150,7 @@ pub struct InferCtxtInner<'tcx> {
     /// for each body-id in this map, which will process the
     /// obligations within. This is expected to be done 'late enough'
     /// that all type inference variables have been bound and so forth.
-    region_obligations: Vec<RegionObligation<'tcx>>,
+    region_obligations: Vec<TypeOutlivesConstraint<'tcx>>,
 
     /// Caches for opaque type inference.
     opaque_type_storage: OpaqueTypeStorage<'tcx>,
@@ -173,7 +173,7 @@ impl<'tcx> InferCtxtInner<'tcx> {
     }
 
     #[inline]
-    pub fn region_obligations(&self) -> &[RegionObligation<'tcx>] {
+    pub fn region_obligations(&self) -> &[TypeOutlivesConstraint<'tcx>] {
         &self.region_obligations
     }
 
@@ -488,7 +488,7 @@ impl fmt::Display for FixupError {
 
 /// See the `region_obligations` field for more information.
 #[derive(Clone, Debug)]
-pub struct RegionObligation<'tcx> {
+pub struct TypeOutlivesConstraint<'tcx> {
     pub sub_region: ty::Region<'tcx>,
     pub sup_type: Ty<'tcx>,
     pub origin: SubregionOrigin<'tcx>,
@@ -738,19 +738,6 @@ impl<'tcx> InferCtxt<'tcx> {
         })
     }
 
-    pub fn region_outlives_predicate(
-        &self,
-        cause: &traits::ObligationCause<'tcx>,
-        predicate: ty::PolyRegionOutlivesPredicate<'tcx>,
-    ) {
-        self.enter_forall(predicate, |ty::OutlivesPredicate(r_a, r_b)| {
-            let origin = SubregionOrigin::from_obligation_cause(cause, || {
-                RelateRegionParamBound(cause.span, None)
-            });
-            self.sub_regions(origin, r_b, r_a); // `b : a` ==> `a <= b`
-        })
-    }
-
     /// Number of type variables created so far.
     pub fn num_ty_vars(&self) -> usize {
         self.inner.borrow_mut().type_variables().num_vars()
@@ -836,6 +823,13 @@ impl<'tcx> InferCtxt<'tcx> {
         ty::Region::new_var(self.tcx, region_var)
     }
 
+    pub fn next_term_var_of_kind(&self, term: ty::Term<'tcx>, span: Span) -> ty::Term<'tcx> {
+        match term.kind() {
+            ty::TermKind::Ty(_) => self.next_ty_var(span).into(),
+            ty::TermKind::Const(_) => self.next_const_var(span).into(),
+        }
+    }
+
     /// Return the universe that the region `r` was created in. For
     /// most regions (e.g., `'static`, named regions from the user,
     /// etc) this is the root universe U0. For inference variables or
@@ -1385,7 +1379,7 @@ impl<'tcx> TyOrConstInferVar {
     /// for types other than `ty::Infer(_)` (or `InferTy::Fresh*`) and
     /// for constants other than `ty::ConstKind::Infer(_)` (or `InferConst::Fresh`).
     pub fn maybe_from_generic_arg(arg: GenericArg<'tcx>) -> Option<Self> {
-        match arg.unpack() {
+        match arg.kind() {
             GenericArgKind::Type(ty) => Self::maybe_from_ty(ty),
             GenericArgKind::Const(ct) => Self::maybe_from_const(ct),
             GenericArgKind::Lifetime(_) => None,
@@ -1396,7 +1390,7 @@ impl<'tcx> TyOrConstInferVar {
     /// for types other than `ty::Infer(_)` (or `InferTy::Fresh*`) and
     /// for constants other than `ty::ConstKind::Infer(_)` (or `InferConst::Fresh`).
     pub fn maybe_from_term(term: Term<'tcx>) -> Option<Self> {
-        match term.unpack() {
+        match term.kind() {
             TermKind::Ty(ty) => Self::maybe_from_ty(ty),
             TermKind::Const(ct) => Self::maybe_from_const(ct),
         }
diff --git a/compiler/rustc_infer/src/infer/opaque_types/table.rs b/compiler/rustc_infer/src/infer/opaque_types/table.rs
index 46752840e1b..df5a66243fc 100644
--- a/compiler/rustc_infer/src/infer/opaque_types/table.rs
+++ b/compiler/rustc_infer/src/infer/opaque_types/table.rs
@@ -24,6 +24,12 @@ pub struct OpaqueTypeStorageEntries {
     duplicate_entries: usize,
 }
 
+impl rustc_type_ir::inherent::OpaqueTypeStorageEntries for OpaqueTypeStorageEntries {
+    fn needs_reevaluation(self, canonicalized: usize) -> bool {
+        self.opaque_types != canonicalized
+    }
+}
+
 impl<'tcx> OpaqueTypeStorage<'tcx> {
     #[instrument(level = "debug")]
     pub(crate) fn remove(
@@ -47,7 +53,7 @@ impl<'tcx> OpaqueTypeStorage<'tcx> {
         assert!(entry.is_some());
     }
 
-    pub(crate) fn is_empty(&self) -> bool {
+    pub fn is_empty(&self) -> bool {
         let OpaqueTypeStorage { opaque_types, duplicate_entries } = self;
         opaque_types.is_empty() && duplicate_entries.is_empty()
     }
diff --git a/compiler/rustc_infer/src/infer/outlives/obligations.rs b/compiler/rustc_infer/src/infer/outlives/obligations.rs
index 5fd98e35e5c..10827c9fd03 100644
--- a/compiler/rustc_infer/src/infer/outlives/obligations.rs
+++ b/compiler/rustc_infer/src/infer/outlives/obligations.rs
@@ -76,23 +76,56 @@ use crate::infer::outlives::env::RegionBoundPairs;
 use crate::infer::outlives::verify::VerifyBoundCx;
 use crate::infer::resolve::OpportunisticRegionResolver;
 use crate::infer::snapshot::undo_log::UndoLog;
-use crate::infer::{self, GenericKind, InferCtxt, RegionObligation, SubregionOrigin, VerifyBound};
+use crate::infer::{
+    self, GenericKind, InferCtxt, SubregionOrigin, TypeOutlivesConstraint, VerifyBound,
+};
 use crate::traits::{ObligationCause, ObligationCauseCode};
 
 impl<'tcx> InferCtxt<'tcx> {
+    pub fn register_outlives_constraint(
+        &self,
+        ty::OutlivesPredicate(arg, r2): ty::OutlivesPredicate<'tcx, ty::GenericArg<'tcx>>,
+        cause: &ObligationCause<'tcx>,
+    ) {
+        match arg.kind() {
+            ty::GenericArgKind::Lifetime(r1) => {
+                self.register_region_outlives_constraint(ty::OutlivesPredicate(r1, r2), cause);
+            }
+            ty::GenericArgKind::Type(ty1) => {
+                self.register_type_outlives_constraint(ty1, r2, cause);
+            }
+            ty::GenericArgKind::Const(_) => unreachable!(),
+        }
+    }
+
+    pub fn register_region_outlives_constraint(
+        &self,
+        ty::OutlivesPredicate(r_a, r_b): ty::RegionOutlivesPredicate<'tcx>,
+        cause: &ObligationCause<'tcx>,
+    ) {
+        let origin = SubregionOrigin::from_obligation_cause(cause, || {
+            SubregionOrigin::RelateRegionParamBound(cause.span, None)
+        });
+        // `'a: 'b` ==> `'b <= 'a`
+        self.sub_regions(origin, r_b, r_a);
+    }
+
     /// Registers that the given region obligation must be resolved
     /// from within the scope of `body_id`. These regions are enqueued
     /// and later processed by regionck, when full type information is
     /// available (see `region_obligations` field for more
     /// information).
     #[instrument(level = "debug", skip(self))]
-    pub fn register_region_obligation(&self, obligation: RegionObligation<'tcx>) {
+    pub fn register_type_outlives_constraint_inner(
+        &self,
+        obligation: TypeOutlivesConstraint<'tcx>,
+    ) {
         let mut inner = self.inner.borrow_mut();
-        inner.undo_log.push(UndoLog::PushRegionObligation);
+        inner.undo_log.push(UndoLog::PushTypeOutlivesConstraint);
         inner.region_obligations.push(obligation);
     }
 
-    pub fn register_region_obligation_with_cause(
+    pub fn register_type_outlives_constraint(
         &self,
         sup_type: Ty<'tcx>,
         sub_region: Region<'tcx>,
@@ -124,11 +157,15 @@ impl<'tcx> InferCtxt<'tcx> {
             )
         });
 
-        self.register_region_obligation(RegionObligation { sup_type, sub_region, origin });
+        self.register_type_outlives_constraint_inner(TypeOutlivesConstraint {
+            sup_type,
+            sub_region,
+            origin,
+        });
     }
 
     /// Trait queries just want to pass back type obligations "as is"
-    pub fn take_registered_region_obligations(&self) -> Vec<RegionObligation<'tcx>> {
+    pub fn take_registered_region_obligations(&self) -> Vec<TypeOutlivesConstraint<'tcx>> {
         std::mem::take(&mut self.inner.borrow_mut().region_obligations)
     }
 
@@ -166,7 +203,7 @@ impl<'tcx> InferCtxt<'tcx> {
                 );
             }
 
-            for RegionObligation { sup_type, sub_region, origin } in my_region_obligations {
+            for TypeOutlivesConstraint { sup_type, sub_region, origin } in my_region_obligations {
                 let outlives = ty::Binder::dummy(ty::OutlivesPredicate(sup_type, sub_region));
                 let ty::OutlivesPredicate(sup_type, sub_region) =
                     deeply_normalize_ty(outlives, origin.clone())
@@ -466,8 +503,8 @@ where
         opt_variances: Option<&[ty::Variance]>,
     ) {
         let constraint = origin.to_constraint_category();
-        for (index, k) in args.iter().enumerate() {
-            match k.unpack() {
+        for (index, arg) in args.iter().enumerate() {
+            match arg.kind() {
                 GenericArgKind::Lifetime(lt) => {
                     let variance = if let Some(variances) = opt_variances {
                         variances[index]
diff --git a/compiler/rustc_infer/src/infer/relate/generalize.rs b/compiler/rustc_infer/src/infer/relate/generalize.rs
index 210b8f37d88..0e17b100276 100644
--- a/compiler/rustc_infer/src/infer/relate/generalize.rs
+++ b/compiler/rustc_infer/src/infer/relate/generalize.rs
@@ -329,7 +329,7 @@ struct Generalizer<'me, 'tcx> {
 impl<'tcx> Generalizer<'_, 'tcx> {
     /// Create an error that corresponds to the term kind in `root_term`
     fn cyclic_term_error(&self) -> TypeError<'tcx> {
-        match self.root_term.unpack() {
+        match self.root_term.kind() {
             ty::TermKind::Ty(ty) => TypeError::CyclicTy(ty),
             ty::TermKind::Const(ct) => TypeError::CyclicConst(ct),
         }
diff --git a/compiler/rustc_infer/src/infer/resolve.rs b/compiler/rustc_infer/src/infer/resolve.rs
index 4b0ace8c554..a95f24b5b95 100644
--- a/compiler/rustc_infer/src/infer/resolve.rs
+++ b/compiler/rustc_infer/src/infer/resolve.rs
@@ -55,6 +55,14 @@ impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for OpportunisticVarResolver<'a, 'tcx> {
             ct.super_fold_with(self)
         }
     }
+
+    fn fold_predicate(&mut self, p: ty::Predicate<'tcx>) -> ty::Predicate<'tcx> {
+        if !p.has_non_region_infer() { p } else { p.super_fold_with(self) }
+    }
+
+    fn fold_clauses(&mut self, c: ty::Clauses<'tcx>) -> ty::Clauses<'tcx> {
+        if !c.has_non_region_infer() { c } else { c.super_fold_with(self) }
+    }
 }
 
 /// The opportunistic region resolver opportunistically resolves regions
diff --git a/compiler/rustc_infer/src/infer/snapshot/undo_log.rs b/compiler/rustc_infer/src/infer/snapshot/undo_log.rs
index b7412d3d6a6..6193f35f3eb 100644
--- a/compiler/rustc_infer/src/infer/snapshot/undo_log.rs
+++ b/compiler/rustc_infer/src/infer/snapshot/undo_log.rs
@@ -26,7 +26,7 @@ pub(crate) enum UndoLog<'tcx> {
     RegionConstraintCollector(region_constraints::UndoLog<'tcx>),
     RegionUnificationTable(sv::UndoLog<ut::Delegate<RegionVidKey<'tcx>>>),
     ProjectionCache(traits::UndoLog<'tcx>),
-    PushRegionObligation,
+    PushTypeOutlivesConstraint,
 }
 
 macro_rules! impl_from {
@@ -72,7 +72,7 @@ impl<'tcx> Rollback<UndoLog<'tcx>> for InferCtxtInner<'tcx> {
                 self.region_constraint_storage.as_mut().unwrap().unification_table.reverse(undo)
             }
             UndoLog::ProjectionCache(undo) => self.projection_cache.reverse(undo),
-            UndoLog::PushRegionObligation => {
+            UndoLog::PushTypeOutlivesConstraint => {
                 self.region_obligations.pop();
             }
         }
diff --git a/compiler/rustc_infer/src/lib.rs b/compiler/rustc_infer/src/lib.rs
index ab7b7060c09..550707ed4bc 100644
--- a/compiler/rustc_infer/src/lib.rs
+++ b/compiler/rustc_infer/src/lib.rs
@@ -20,7 +20,6 @@
 #![doc(rust_logo)]
 #![feature(assert_matches)]
 #![feature(extend_one)]
-#![feature(iterator_try_collect)]
 #![feature(rustdoc_internals)]
 #![recursion_limit = "512"] // For rustdoc
 // tidy-alphabetical-end
diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index cf494f8d686..e824e9d4aa9 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -18,7 +18,7 @@ use rustc_parse::parser::attr::AllowLeadingUnsafe;
 use rustc_query_impl::QueryCtxt;
 use rustc_query_system::query::print_query_stack;
 use rustc_session::config::{self, Cfg, CheckCfg, ExpectedValues, Input, OutFileName};
-use rustc_session::filesearch::sysroot_candidates;
+use rustc_session::filesearch::sysroot_with_fallback;
 use rustc_session::parse::ParseSess;
 use rustc_session::{CompilerIO, EarlyDiagCtxt, Session, lint};
 use rustc_span::source_map::{FileLoader, RealFileLoader, SourceMapInputs};
@@ -442,8 +442,7 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se
             let temps_dir = config.opts.unstable_opts.temps_dir.as_deref().map(PathBuf::from);
 
             let bundle = match rustc_errors::fluent_bundle(
-                config.opts.sysroot.clone(),
-                sysroot_candidates().to_vec(),
+                sysroot_with_fallback(&config.opts.sysroot),
                 config.opts.unstable_opts.translate_lang.clone(),
                 config.opts.unstable_opts.translate_additional_ftl.as_deref(),
                 config.opts.unstable_opts.translate_directionality_markers,
diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs
index 8f6c5b47ee2..2643e5c1926 100644
--- a/compiler/rustc_interface/src/passes.rs
+++ b/compiler/rustc_interface/src/passes.rs
@@ -45,7 +45,7 @@ use crate::interface::Compiler;
 use crate::{errors, limits, proc_macro_decls, util};
 
 pub fn parse<'a>(sess: &'a Session) -> ast::Crate {
-    let krate = sess
+    let mut krate = sess
         .time("parse_crate", || {
             let mut parser = unwrap_or_emit_fatal(match &sess.io.input {
                 Input::File(file) => new_parser_from_file(&sess.psess, file, None),
@@ -60,9 +60,11 @@ pub fn parse<'a>(sess: &'a Session) -> ast::Crate {
             guar.raise_fatal();
         });
 
-    if sess.opts.unstable_opts.input_stats {
-        input_stats::print_ast_stats(&krate, "PRE EXPANSION AST STATS", "ast-stats-1");
-    }
+    rustc_builtin_macros::cmdline_attrs::inject(
+        &mut krate,
+        &sess.psess,
+        &sess.opts.unstable_opts.crate_attr,
+    );
 
     krate
 }
@@ -292,7 +294,7 @@ fn early_lint_checks(tcx: TyCtxt<'_>, (): ()) {
     let mut lint_buffer = resolver.lint_buffer.steal();
 
     if sess.opts.unstable_opts.input_stats {
-        input_stats::print_ast_stats(krate, "POST EXPANSION AST STATS", "ast-stats-2");
+        input_stats::print_ast_stats(krate, "POST EXPANSION AST STATS", "ast-stats");
     }
 
     // Needs to go *after* expansion to be able to check the results of macro expansion.
@@ -805,17 +807,11 @@ pub static DEFAULT_QUERY_PROVIDERS: LazyLock<Providers> = LazyLock::new(|| {
 
 pub fn create_and_enter_global_ctxt<T, F: for<'tcx> FnOnce(TyCtxt<'tcx>) -> T>(
     compiler: &Compiler,
-    mut krate: rustc_ast::Crate,
+    krate: rustc_ast::Crate,
     f: F,
 ) -> T {
     let sess = &compiler.sess;
 
-    rustc_builtin_macros::cmdline_attrs::inject(
-        &mut krate,
-        &sess.psess,
-        &sess.opts.unstable_opts.crate_attr,
-    );
-
     let pre_configured_attrs = rustc_expand::config::pre_configure_attrs(sess, &krate.attrs);
 
     let crate_name = get_crate_name(sess, &pre_configured_attrs);
@@ -960,7 +956,6 @@ fn run_required_analyses(tcx: TyCtxt<'_>) {
                 tcx.par_hir_for_each_module(|module| {
                     tcx.ensure_ok().check_mod_loops(module);
                     tcx.ensure_ok().check_mod_attrs(module);
-                    tcx.ensure_ok().check_mod_naked_functions(module);
                     tcx.ensure_ok().check_mod_unstable_api_usage(module);
                 });
             },
@@ -981,13 +976,6 @@ fn run_required_analyses(tcx: TyCtxt<'_>) {
     });
 
     rustc_hir_analysis::check_crate(tcx);
-    sess.time("MIR_coroutine_by_move_body", || {
-        tcx.par_hir_body_owners(|def_id| {
-            if tcx.needs_coroutine_by_move_body_def_id(def_id.to_def_id()) {
-                tcx.ensure_done().coroutine_by_move_body_def_id(def_id);
-            }
-        });
-    });
     // Freeze definitions as we don't add new ones at this point.
     // We need to wait until now since we synthesize a by-move body
     // for all coroutine-closures.
@@ -1012,10 +1000,6 @@ fn run_required_analyses(tcx: TyCtxt<'_>) {
             {
                 tcx.ensure_ok().mir_drops_elaborated_and_const_checked(def_id);
             }
-        });
-    });
-    sess.time("coroutine_obligations", || {
-        tcx.par_hir_body_owners(|def_id| {
             if tcx.is_coroutine(def_id.to_def_id()) {
                 tcx.ensure_ok().mir_coroutine_witnesses(def_id);
                 let _ = tcx.ensure_ok().check_coroutine_obligations(
diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs
index 20e081d3360..558f13a832c 100644
--- a/compiler/rustc_interface/src/tests.rs
+++ b/compiler/rustc_interface/src/tests.rs
@@ -715,6 +715,7 @@ fn test_unstable_options_tracking_hash() {
     untracked!(no_analysis, true);
     untracked!(no_leak_check, true);
     untracked!(no_parallel_backend, true);
+    untracked!(no_steal_thir, true);
     untracked!(parse_crate_root_only, true);
     // `pre_link_arg` is omitted because it just forwards to `pre_link_args`.
     untracked!(pre_link_args, vec![String::from("abc"), String::from("def")]);
@@ -776,8 +777,7 @@ fn test_unstable_options_tracking_hash() {
         CoverageOptions {
             level: CoverageLevel::Mcdc,
             no_mir_spans: true,
-            discard_all_spans_in_codegen: true,
-            inject_unused_local_file: true,
+            discard_all_spans_in_codegen: true
         }
     );
     tracked!(crate_attr, vec!["abc".to_string()]);
diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs
index 087b11fdf9d..8bdc24d47d9 100644
--- a/compiler/rustc_interface/src/util.rs
+++ b/compiler/rustc_interface/src/util.rs
@@ -2,7 +2,7 @@ use std::env::consts::{DLL_PREFIX, DLL_SUFFIX};
 use std::path::{Path, PathBuf};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, OnceLock};
-use std::{env, iter, thread};
+use std::{env, thread};
 
 use rustc_ast as ast;
 use rustc_codegen_ssa::traits::CodegenBackend;
@@ -12,7 +12,6 @@ use rustc_metadata::{DylibError, load_symbol_from_dylib};
 use rustc_middle::ty::CurrentGcx;
 use rustc_parse::validate_attr;
 use rustc_session::config::{Cfg, OutFileName, OutputFilenames, OutputTypes, host_tuple};
-use rustc_session::filesearch::sysroot_candidates;
 use rustc_session::lint::{self, BuiltinLintDiag, LintBuffer};
 use rustc_session::output::{CRATE_TYPES, categorize_crate_type};
 use rustc_session::{EarlyDiagCtxt, Session, filesearch};
@@ -346,14 +345,10 @@ pub fn rustc_path<'a>() -> Option<&'a Path> {
 }
 
 fn get_rustc_path_inner(bin_path: &str) -> Option<PathBuf> {
-    sysroot_candidates().iter().find_map(|sysroot| {
-        let candidate = sysroot.join(bin_path).join(if cfg!(target_os = "windows") {
-            "rustc.exe"
-        } else {
-            "rustc"
-        });
-        candidate.exists().then_some(candidate)
-    })
+    let candidate = filesearch::get_or_default_sysroot()
+        .join(bin_path)
+        .join(if cfg!(target_os = "windows") { "rustc.exe" } else { "rustc" });
+    candidate.exists().then_some(candidate)
 }
 
 #[allow(rustc::untranslatable_diagnostic)] // FIXME: make this translatable
@@ -374,10 +369,10 @@ fn get_codegen_sysroot(
     );
 
     let target = host_tuple();
-    let sysroot_candidates = sysroot_candidates();
+    let sysroot_candidates = filesearch::sysroot_with_fallback(&sysroot);
 
-    let sysroot = iter::once(sysroot)
-        .chain(sysroot_candidates.iter().map(<_>::as_ref))
+    let sysroot = sysroot_candidates
+        .iter()
         .map(|sysroot| {
             filesearch::make_target_lib_path(sysroot, target).with_file_name("codegen-backends")
         })
diff --git a/compiler/rustc_lexer/src/cursor.rs b/compiler/rustc_lexer/src/cursor.rs
index 526693d3de1..165262b82c7 100644
--- a/compiler/rustc_lexer/src/cursor.rs
+++ b/compiler/rustc_lexer/src/cursor.rs
@@ -68,7 +68,7 @@ impl<'a> Cursor<'a> {
 
     /// Peeks the third symbol from the input stream without consuming it.
     pub fn third(&self) -> char {
-        // `.next()` optimizes better than `.nth(1)`
+        // `.next()` optimizes better than `.nth(2)`
         let mut iter = self.chars.clone();
         iter.next();
         iter.next();
diff --git a/compiler/rustc_lexer/src/lib.rs b/compiler/rustc_lexer/src/lib.rs
index 2374f388250..b2bd5e188ef 100644
--- a/compiler/rustc_lexer/src/lib.rs
+++ b/compiler/rustc_lexer/src/lib.rs
@@ -30,14 +30,13 @@ mod cursor;
 #[cfg(test)]
 mod tests;
 
+use LiteralKind::*;
+use TokenKind::*;
+use cursor::EOF_CHAR;
+pub use cursor::{Cursor, FrontmatterAllowed};
 use unicode_properties::UnicodeEmoji;
 pub use unicode_xid::UNICODE_VERSION as UNICODE_XID_VERSION;
 
-use self::LiteralKind::*;
-use self::TokenKind::*;
-use crate::cursor::EOF_CHAR;
-pub use crate::cursor::{Cursor, FrontmatterAllowed};
-
 /// Parsed token.
 /// It doesn't contain information about data that has been parsed,
 /// only the type of the token and its size.
@@ -372,9 +371,8 @@ pub fn is_ident(string: &str) -> bool {
 impl Cursor<'_> {
     /// Parses a token from the input string.
     pub fn advance_token(&mut self) -> Token {
-        let first_char = match self.bump() {
-            Some(c) => c,
-            None => return Token::new(TokenKind::Eof, 0),
+        let Some(first_char) = self.bump() else {
+            return Token::new(TokenKind::Eof, 0);
         };
 
         let token_kind = match first_char {
@@ -545,11 +543,12 @@ impl Cursor<'_> {
 
         let mut s = self.as_str();
         let mut found = false;
+        let mut size = 0;
         while let Some(closing) = s.find(&"-".repeat(length_opening as usize)) {
             let preceding_chars_start = s[..closing].rfind("\n").map_or(0, |i| i + 1);
             if s[preceding_chars_start..closing].chars().all(is_whitespace) {
                 // candidate found
-                self.bump_bytes(closing);
+                self.bump_bytes(size + closing);
                 // in case like
                 // ---cargo
                 // --- blahblah
@@ -562,6 +561,7 @@ impl Cursor<'_> {
                 break;
             } else {
                 s = &s[closing + length_opening as usize..];
+                size += closing + length_opening as usize;
             }
         }
 
@@ -786,7 +786,7 @@ impl Cursor<'_> {
         } else {
             // No base prefix, parse number in the usual way.
             self.eat_decimal_digits();
-        };
+        }
 
         match self.first() {
             // Don't be greedy if this is actually an
diff --git a/compiler/rustc_lint/messages.ftl b/compiler/rustc_lint/messages.ftl
index 08180bf8f8b..fd2e2ba39ac 100644
--- a/compiler/rustc_lint/messages.ftl
+++ b/compiler/rustc_lint/messages.ftl
@@ -13,6 +13,8 @@ lint_ambiguous_negative_literals = `-` has lower precedence than method calls, w
 lint_ambiguous_wide_pointer_comparisons = ambiguous wide pointer comparison, the comparison includes metadata which may not be expected
     .addr_metadata_suggestion = use explicit `std::ptr::eq` method to compare metadata and addresses
     .addr_suggestion = use `std::ptr::addr_eq` or untyped pointers to only compare their addresses
+    .cast_suggestion = use untyped pointers to only compare their addresses
+    .expect_suggestion = or expect the lint to compare the pointers metadata and addresses
 
 lint_associated_const_elided_lifetime = {$elided ->
         [true] `&` without an explicit lifetime name cannot be used here
@@ -251,11 +253,6 @@ lint_duplicate_macro_attribute =
 
 lint_duplicate_matcher_binding = duplicate matcher binding
 
-lint_elided_named_lifetime = elided lifetime has a name
-    .label_elided = this elided lifetime gets resolved as `{$name}`
-    .label_named = lifetime `{$name}` declared here
-    .suggestion = consider specifying it explicitly
-
 lint_enum_intrinsics_mem_discriminant =
     the return value of `mem::discriminant` is unspecified when called with a non-enum type
     .note = the argument to `discriminant` should be a reference to an enum, but it was passed a reference to a `{$ty_param}`, which is not an enum
@@ -372,8 +369,6 @@ lint_improper_ctypes = `extern` {$desc} uses type `{$ty}`, which is not FFI-safe
     .label = not FFI-safe
     .note = the type is defined here
 
-lint_improper_ctypes_128bit = 128-bit integers don't currently have a known stable ABI
-
 lint_improper_ctypes_array_help = consider passing a pointer to the array
 
 lint_improper_ctypes_array_reason = passing raw arrays by value is not FFI-safe
@@ -516,6 +511,28 @@ lint_metavariable_still_repeating = variable `{$name}` is still repeating at thi
 
 lint_metavariable_wrong_operator = meta-variable repeats with different Kleene operator
 
+lint_mismatched_lifetime_syntaxes =
+    lifetime flowing from input to output with different syntax can be confusing
+    .label_mismatched_lifetime_syntaxes_inputs =
+        {$n_inputs ->
+            [one] this lifetime flows
+            *[other] these lifetimes flow
+        } to the output
+    .label_mismatched_lifetime_syntaxes_outputs =
+        the {$n_outputs ->
+            [one] lifetime gets
+            *[other] lifetimes get
+        } resolved as `{$lifetime_name}`
+
+lint_mismatched_lifetime_syntaxes_suggestion_explicit =
+    one option is to consistently use `{$lifetime_name}`
+
+lint_mismatched_lifetime_syntaxes_suggestion_implicit =
+    one option is to consistently remove the lifetime
+
+lint_mismatched_lifetime_syntaxes_suggestion_mixed =
+    one option is to remove the lifetime for references and use the anonymous lifetime for paths
+
 lint_missing_fragment_specifier = missing fragment specifier
 
 lint_missing_unsafe_on_extern = extern blocks should be unsafe
@@ -805,6 +822,11 @@ lint_type_ir_inherent_usage = do not use `rustc_type_ir::inherent` unless you're
 lint_type_ir_trait_usage = do not use `rustc_type_ir::Interner` or `rustc_type_ir::InferCtxtLike` unless you're inside of the trait solver
     .note = the method or struct you're looking for is likely defined somewhere else downstream in the compiler
 
+lint_undefined_transmute = pointers cannot be transmuted to integers during const eval
+    .note = at compile-time, pointers do not have an integer value
+    .note2 = avoiding this restriction via `union` or raw pointers leads to compile-time undefined behavior
+    .help = for more information, see https://doc.rust-lang.org/std/mem/fn.transmute.html
+
 lint_undropped_manually_drops = calls to `std::mem::drop` with `std::mem::ManuallyDrop` instead of the inner value does nothing
     .label = argument has type `{$arg_ty}`
     .suggestion = use `std::mem::ManuallyDrop::into_inner` to get the inner value
@@ -835,6 +857,7 @@ lint_unexpected_cfg_name_similar_name = there is a config with a similar name
 lint_unexpected_cfg_name_similar_name_different_values = there is a config with a similar name and different values
 lint_unexpected_cfg_name_similar_name_no_value = there is a config with a similar name and no value
 lint_unexpected_cfg_name_similar_name_value = there is a config with a similar name and value
+lint_unexpected_cfg_name_version_syntax = there is a similar config predicate: `version("..")`
 lint_unexpected_cfg_name_with_similar_value = found config with similar value
 
 lint_unexpected_cfg_value = unexpected `cfg` condition value: {$has_value ->
@@ -952,7 +975,8 @@ lint_unused_doc_comment = unused doc comment
     .help = to document an item produced by a macro, the macro must produce the documentation as part of its expansion
 
 lint_unused_extern_crate = unused extern crate
-    .suggestion = remove it
+    .label = unused
+    .suggestion = remove the unused `extern crate`
 
 lint_unused_import_braces = braces around {$node} is unnecessary
 
diff --git a/compiler/rustc_lint/src/autorefs.rs b/compiler/rustc_lint/src/autorefs.rs
index 5de2cbf9939..845a1f1b81f 100644
--- a/compiler/rustc_lint/src/autorefs.rs
+++ b/compiler/rustc_lint/src/autorefs.rs
@@ -16,7 +16,7 @@ declare_lint! {
     ///
     /// ### Example
     ///
-    /// ```rust
+    /// ```rust,compile_fail
     /// unsafe fn fun(ptr: *mut [u8]) -> *mut [u8] {
     ///     unsafe { &raw mut (*ptr)[..16] }
     ///     //                      ^^^^^^ this calls `IndexMut::index_mut(&mut ..., ..16)`,
@@ -51,7 +51,7 @@ declare_lint! {
     /// }
     /// ```
     pub DANGEROUS_IMPLICIT_AUTOREFS,
-    Warn,
+    Deny,
     "implicit reference to a dereference of a raw pointer",
     report_in_external_macro
 }
diff --git a/compiler/rustc_lint/src/builtin.rs b/compiler/rustc_lint/src/builtin.rs
index 95e31e4af1e..69e9f8e1b2c 100644
--- a/compiler/rustc_lint/src/builtin.rs
+++ b/compiler/rustc_lint/src/builtin.rs
@@ -39,7 +39,7 @@ pub use rustc_session::lint::builtin::*;
 use rustc_session::{declare_lint, declare_lint_pass, impl_lint_pass};
 use rustc_span::edition::Edition;
 use rustc_span::source_map::Spanned;
-use rustc_span::{BytePos, Ident, InnerSpan, Span, Symbol, kw, sym};
+use rustc_span::{BytePos, DUMMY_SP, Ident, InnerSpan, Span, Symbol, kw, sym};
 use rustc_target::asm::InlineAsmArch;
 use rustc_trait_selection::infer::{InferCtxtExt, TyCtxtInferExt};
 use rustc_trait_selection::traits::misc::type_allowed_to_implement_copy;
@@ -545,22 +545,22 @@ impl<'tcx> LateLintPass<'tcx> for MissingCopyImplementations {
             return;
         }
         let (def, ty) = match item.kind {
-            hir::ItemKind::Struct(_, _, ast_generics) => {
-                if !ast_generics.params.is_empty() {
+            hir::ItemKind::Struct(_, generics, _) => {
+                if !generics.params.is_empty() {
                     return;
                 }
                 let def = cx.tcx.adt_def(item.owner_id);
                 (def, Ty::new_adt(cx.tcx, def, ty::List::empty()))
             }
-            hir::ItemKind::Union(_, _, ast_generics) => {
-                if !ast_generics.params.is_empty() {
+            hir::ItemKind::Union(_, generics, _) => {
+                if !generics.params.is_empty() {
                     return;
                 }
                 let def = cx.tcx.adt_def(item.owner_id);
                 (def, Ty::new_adt(cx.tcx, def, ty::List::empty()))
             }
-            hir::ItemKind::Enum(_, _, ast_generics) => {
-                if !ast_generics.params.is_empty() {
+            hir::ItemKind::Enum(_, generics, _) => {
+                if !generics.params.is_empty() {
                     return;
                 }
                 let def = cx.tcx.adt_def(item.owner_id);
@@ -635,7 +635,8 @@ fn type_implements_negative_copy_modulo_regions<'tcx>(
     typing_env: ty::TypingEnv<'tcx>,
 ) -> bool {
     let (infcx, param_env) = tcx.infer_ctxt().build_with_typing_env(typing_env);
-    let trait_ref = ty::TraitRef::new(tcx, tcx.require_lang_item(hir::LangItem::Copy, None), [ty]);
+    let trait_ref =
+        ty::TraitRef::new(tcx, tcx.require_lang_item(hir::LangItem::Copy, DUMMY_SP), [ty]);
     let pred = ty::TraitPredicate { trait_ref, polarity: ty::PredicatePolarity::Negative };
     let obligation = traits::Obligation {
         cause: traits::ObligationCause::dummy(),
@@ -1422,7 +1423,7 @@ impl TypeAliasBounds {
 
 impl<'tcx> LateLintPass<'tcx> for TypeAliasBounds {
     fn check_item(&mut self, cx: &LateContext<'_>, item: &hir::Item<'_>) {
-        let hir::ItemKind::TyAlias(_, hir_ty, generics) = item.kind else { return };
+        let hir::ItemKind::TyAlias(_, generics, hir_ty) = item.kind else { return };
 
         // There must not be a where clause.
         if generics.predicates.is_empty() {
@@ -2125,9 +2126,9 @@ impl<'tcx> LateLintPass<'tcx> for ExplicitOutlivesRequirements {
         use rustc_middle::middle::resolve_bound_vars::ResolvedArg;
 
         let def_id = item.owner_id.def_id;
-        if let hir::ItemKind::Struct(_, _, hir_generics)
-        | hir::ItemKind::Enum(_, _, hir_generics)
-        | hir::ItemKind::Union(_, _, hir_generics) = item.kind
+        if let hir::ItemKind::Struct(_, generics, _)
+        | hir::ItemKind::Enum(_, generics, _)
+        | hir::ItemKind::Union(_, generics, _) = item.kind
         {
             let inferred_outlives = cx.tcx.inferred_outlives_of(def_id);
             if inferred_outlives.is_empty() {
@@ -2135,7 +2136,7 @@ impl<'tcx> LateLintPass<'tcx> for ExplicitOutlivesRequirements {
             }
 
             let ty_generics = cx.tcx.generics_of(def_id);
-            let num_where_predicates = hir_generics
+            let num_where_predicates = generics
                 .predicates
                 .iter()
                 .filter(|predicate| predicate.kind.in_where_clause())
@@ -2145,7 +2146,7 @@ impl<'tcx> LateLintPass<'tcx> for ExplicitOutlivesRequirements {
             let mut lint_spans = Vec::new();
             let mut where_lint_spans = Vec::new();
             let mut dropped_where_predicate_count = 0;
-            for (i, where_predicate) in hir_generics.predicates.iter().enumerate() {
+            for (i, where_predicate) in generics.predicates.iter().enumerate() {
                 let (relevant_lifetimes, bounds, predicate_span, in_where_clause) =
                     match where_predicate.kind {
                         hir::WherePredicateKind::RegionPredicate(predicate) => {
@@ -2228,7 +2229,7 @@ impl<'tcx> LateLintPass<'tcx> for ExplicitOutlivesRequirements {
                     } else if i + 1 < num_where_predicates {
                         // If all the bounds on a predicate were inferable and there are
                         // further predicates, we want to eat the trailing comma.
-                        let next_predicate_span = hir_generics.predicates[i + 1].span;
+                        let next_predicate_span = generics.predicates[i + 1].span;
                         if next_predicate_span.from_expansion() {
                             where_lint_spans.push(predicate_span);
                         } else {
@@ -2237,7 +2238,7 @@ impl<'tcx> LateLintPass<'tcx> for ExplicitOutlivesRequirements {
                         }
                     } else {
                         // Eat the optional trailing comma after the last predicate.
-                        let where_span = hir_generics.where_clause_span;
+                        let where_span = generics.where_clause_span;
                         if where_span.from_expansion() {
                             where_lint_spans.push(predicate_span);
                         } else {
@@ -2255,18 +2256,18 @@ impl<'tcx> LateLintPass<'tcx> for ExplicitOutlivesRequirements {
 
             // If all predicates in where clause are inferable, drop the entire clause
             // (including the `where`)
-            if hir_generics.has_where_clause_predicates
+            if generics.has_where_clause_predicates
                 && dropped_where_predicate_count == num_where_predicates
             {
-                let where_span = hir_generics.where_clause_span;
+                let where_span = generics.where_clause_span;
                 // Extend the where clause back to the closing `>` of the
                 // generics, except for tuple struct, which have the `where`
                 // after the fields of the struct.
                 let full_where_span =
-                    if let hir::ItemKind::Struct(_, hir::VariantData::Tuple(..), _) = item.kind {
+                    if let hir::ItemKind::Struct(_, _, hir::VariantData::Tuple(..)) = item.kind {
                         where_span
                     } else {
-                        hir_generics.span.shrink_to_hi().to(where_span)
+                        generics.span.shrink_to_hi().to(where_span)
                     };
 
                 // Due to macro expansions, the `full_where_span` might not actually contain all
diff --git a/compiler/rustc_lint/src/default_could_be_derived.rs b/compiler/rustc_lint/src/default_could_be_derived.rs
index 78d129642dc..7734f441df2 100644
--- a/compiler/rustc_lint/src/default_could_be_derived.rs
+++ b/compiler/rustc_lint/src/default_could_be_derived.rs
@@ -95,8 +95,8 @@ impl<'tcx> LateLintPass<'tcx> for DefaultCouldBeDerived {
                 kind:
                     hir::ItemKind::Struct(
                         _,
-                        hir::VariantData::Struct { fields, recovered: _ },
                         _generics,
+                        hir::VariantData::Struct { fields, recovered: _ },
                     ),
                 ..
             })) => fields.iter().map(|f| (f.ident.name, f)).collect::<FxHashMap<_, _>>(),
diff --git a/compiler/rustc_lint/src/early.rs b/compiler/rustc_lint/src/early.rs
index f9601fa5ef1..4978f293ab7 100644
--- a/compiler/rustc_lint/src/early.rs
+++ b/compiler/rustc_lint/src/early.rs
@@ -18,7 +18,7 @@ use tracing::debug;
 use crate::context::{EarlyContext, LintContext, LintStore};
 use crate::passes::{EarlyLintPass, EarlyLintPassObject};
 
-mod diagnostics;
+pub(super) mod diagnostics;
 
 macro_rules! lint_callback { ($cx:expr, $f:ident, $($args:expr),*) => ({
     $cx.pass.$f(&$cx.context, $($args),*);
@@ -40,7 +40,7 @@ impl<'ecx, 'tcx, T: EarlyLintPass> EarlyContextAndPass<'ecx, 'tcx, T> {
         for early_lint in self.context.buffered.take(id) {
             let BufferedEarlyLint { span, node_id: _, lint_id, diagnostic } = early_lint;
             self.context.opt_span_lint(lint_id.lint, span, |diag| {
-                diagnostics::decorate_lint(self.context.sess(), self.tcx, diagnostic, diag);
+                diagnostics::decorate_builtin_lint(self.context.sess(), self.tcx, diagnostic, diag);
             });
         }
     }
@@ -74,8 +74,8 @@ impl<'ecx, 'tcx, T: EarlyLintPass> EarlyContextAndPass<'ecx, 'tcx, T> {
 impl<'ast, 'ecx, 'tcx, T: EarlyLintPass> ast_visit::Visitor<'ast>
     for EarlyContextAndPass<'ecx, 'tcx, T>
 {
-    fn visit_coroutine_kind(&mut self, coroutine_kind: &'ast ast::CoroutineKind) -> Self::Result {
-        self.check_id(coroutine_kind.closure_id());
+    fn visit_id(&mut self, id: rustc_ast::NodeId) {
+        self.check_id(id);
     }
 
     fn visit_param(&mut self, param: &'ast ast::Param) {
@@ -101,7 +101,6 @@ impl<'ast, 'ecx, 'tcx, T: EarlyLintPass> ast_visit::Visitor<'ast>
 
     fn visit_pat(&mut self, p: &'ast ast::Pat) {
         lint_callback!(self, check_pat, p);
-        self.check_id(p.id);
         ast_visit::walk_pat(self, p);
         lint_callback!(self, check_pat_post, p);
     }
@@ -112,11 +111,6 @@ impl<'ast, 'ecx, 'tcx, T: EarlyLintPass> ast_visit::Visitor<'ast>
         });
     }
 
-    fn visit_anon_const(&mut self, c: &'ast ast::AnonConst) {
-        self.check_id(c.id);
-        ast_visit::walk_anon_const(self, c);
-    }
-
     fn visit_expr(&mut self, e: &'ast ast::Expr) {
         self.with_lint_attrs(e.id, &e.attrs, |cx| {
             lint_callback!(cx, check_expr, e);
@@ -157,13 +151,6 @@ impl<'ast, 'ecx, 'tcx, T: EarlyLintPass> ast_visit::Visitor<'ast>
         ast_visit::walk_fn(self, fk);
     }
 
-    fn visit_variant_data(&mut self, s: &'ast ast::VariantData) {
-        if let Some(ctor_node_id) = s.ctor_node_id() {
-            self.check_id(ctor_node_id);
-        }
-        ast_visit::walk_struct_def(self, s);
-    }
-
     fn visit_field_def(&mut self, s: &'ast ast::FieldDef) {
         self.with_lint_attrs(s.id, &s.attrs, |cx| {
             ast_visit::walk_field_def(cx, s);
@@ -179,7 +166,6 @@ impl<'ast, 'ecx, 'tcx, T: EarlyLintPass> ast_visit::Visitor<'ast>
 
     fn visit_ty(&mut self, t: &'ast ast::Ty) {
         lint_callback!(self, check_ty, t);
-        self.check_id(t.id);
         ast_visit::walk_ty(self, t);
     }
 
@@ -196,7 +182,6 @@ impl<'ast, 'ecx, 'tcx, T: EarlyLintPass> ast_visit::Visitor<'ast>
 
     fn visit_block(&mut self, b: &'ast ast::Block) {
         lint_callback!(self, check_block, b);
-        self.check_id(b.id);
         ast_visit::walk_block(self, b);
     }
 
@@ -257,29 +242,13 @@ impl<'ast, 'ecx, 'tcx, T: EarlyLintPass> ast_visit::Visitor<'ast>
         });
     }
 
-    fn visit_lifetime(&mut self, lt: &'ast ast::Lifetime, _: ast_visit::LifetimeCtxt) {
-        self.check_id(lt.id);
-        ast_visit::walk_lifetime(self, lt);
-    }
-
-    fn visit_path(&mut self, p: &'ast ast::Path, id: ast::NodeId) {
-        self.check_id(id);
-        ast_visit::walk_path(self, p);
-    }
-
-    fn visit_path_segment(&mut self, s: &'ast ast::PathSegment) {
-        self.check_id(s.id);
-        ast_visit::walk_path_segment(self, s);
-    }
-
     fn visit_attribute(&mut self, attr: &'ast ast::Attribute) {
         lint_callback!(self, check_attribute, attr);
         ast_visit::walk_attribute(self, attr);
     }
 
-    fn visit_mac_def(&mut self, mac: &'ast ast::MacroDef, id: ast::NodeId) {
+    fn visit_macro_def(&mut self, mac: &'ast ast::MacroDef) {
         lint_callback!(self, check_mac_def, mac);
-        self.check_id(id);
     }
 
     fn visit_mac_call(&mut self, mac: &'ast ast::MacCall) {
diff --git a/compiler/rustc_lint/src/early/diagnostics.rs b/compiler/rustc_lint/src/early/diagnostics.rs
index 40ca9e05d95..60c477dd6c7 100644
--- a/compiler/rustc_lint/src/early/diagnostics.rs
+++ b/compiler/rustc_lint/src/early/diagnostics.rs
@@ -10,15 +10,15 @@ use rustc_errors::{
 use rustc_middle::middle::stability;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::Session;
-use rustc_session::lint::{BuiltinLintDiag, ElidedLifetimeResolution};
-use rustc_span::{BytePos, kw};
+use rustc_session::lint::BuiltinLintDiag;
+use rustc_span::BytePos;
 use tracing::debug;
 
-use crate::lints::{self, ElidedNamedLifetime};
+use crate::lints;
 
 mod check_cfg;
 
-pub(super) fn decorate_lint(
+pub fn decorate_builtin_lint(
     sess: &Session,
     tcx: Option<TyCtxt<'_>>,
     diagnostic: BuiltinLintDiag,
@@ -187,6 +187,27 @@ pub(super) fn decorate_lint(
                 lints::ReservedMultihash { suggestion }.decorate_lint(diag);
             }
         }
+        BuiltinLintDiag::HiddenUnicodeCodepoints {
+            label,
+            count,
+            span_label,
+            labels,
+            escape,
+            spans,
+        } => {
+            lints::HiddenUnicodeCodepointsDiag {
+                label: &label,
+                count,
+                span_label,
+                labels: labels.map(|spans| lints::HiddenUnicodeCodepointsDiagLabels { spans }),
+                sub: if escape {
+                    lints::HiddenUnicodeCodepointsDiagSub::Escape { spans }
+                } else {
+                    lints::HiddenUnicodeCodepointsDiagSub::NoEscape { spans }
+                },
+            }
+            .decorate_lint(diag);
+        }
         BuiltinLintDiag::UnusedBuiltinAttribute { attr_name, macro_name, invoc_span } => {
             lints::UnusedBuiltinAttribute { invoc_span, attr_name, macro_name }.decorate_lint(diag);
         }
@@ -292,8 +313,8 @@ pub(super) fn decorate_lint(
         BuiltinLintDiag::ByteSliceInPackedStructWithDerive { ty } => {
             lints::ByteSliceInPackedStructWithDerive { ty }.decorate_lint(diag);
         }
-        BuiltinLintDiag::UnusedExternCrate { removal_span } => {
-            lints::UnusedExternCrate { removal_span }.decorate_lint(diag);
+        BuiltinLintDiag::UnusedExternCrate { span, removal_span } => {
+            lints::UnusedExternCrate { span, removal_span }.decorate_lint(diag);
         }
         BuiltinLintDiag::ExternCrateNotIdiomatic { vis_span, ident_span } => {
             let suggestion_span = vis_span.between(ident_span);
@@ -450,16 +471,5 @@ pub(super) fn decorate_lint(
         BuiltinLintDiag::UnexpectedBuiltinCfg { cfg, cfg_name, controlled_by } => {
             lints::UnexpectedBuiltinCfg { cfg, cfg_name, controlled_by }.decorate_lint(diag)
         }
-        BuiltinLintDiag::ElidedNamedLifetimes { elided: (span, kind), resolution } => {
-            match resolution {
-                ElidedLifetimeResolution::Static => {
-                    ElidedNamedLifetime { span, kind, name: kw::StaticLifetime, declaration: None }
-                }
-                ElidedLifetimeResolution::Param(name, declaration) => {
-                    ElidedNamedLifetime { span, kind, name, declaration: Some(declaration) }
-                }
-            }
-            .decorate_lint(diag)
-        }
     }
 }
diff --git a/compiler/rustc_lint/src/early/diagnostics/check_cfg.rs b/compiler/rustc_lint/src/early/diagnostics/check_cfg.rs
index 946dbc34f71..e2f5dd315d5 100644
--- a/compiler/rustc_lint/src/early/diagnostics/check_cfg.rs
+++ b/compiler/rustc_lint/src/early/diagnostics/check_cfg.rs
@@ -140,6 +140,14 @@ pub(super) fn unexpected_cfg_name(
 
     let code_sugg = if is_feature_cfg && is_from_cargo {
         lints::unexpected_cfg_name::CodeSuggestion::DefineFeatures
+    // Suggest correct `version("..")` predicate syntax
+    } else if let Some((_value, value_span)) = value
+        && name == sym::version
+    {
+        lints::unexpected_cfg_name::CodeSuggestion::VersionSyntax {
+            between_name_and_value: name_span.between(value_span),
+            after_value: value_span.shrink_to_hi(),
+        }
     // Suggest the most probable if we found one
     } else if let Some(best_match) = find_best_match_for_name(&possibilities, name, None) {
         is_feature_cfg |= best_match == sym::feature;
diff --git a/compiler/rustc_lint/src/hidden_unicode_codepoints.rs b/compiler/rustc_lint/src/hidden_unicode_codepoints.rs
deleted file mode 100644
index 491c2826baa..00000000000
--- a/compiler/rustc_lint/src/hidden_unicode_codepoints.rs
+++ /dev/null
@@ -1,136 +0,0 @@
-use ast::util::unicode::{TEXT_FLOW_CONTROL_CHARS, contains_text_flow_control_chars};
-use rustc_ast as ast;
-use rustc_session::{declare_lint, declare_lint_pass};
-use rustc_span::{BytePos, Span, Symbol};
-
-use crate::lints::{
-    HiddenUnicodeCodepointsDiag, HiddenUnicodeCodepointsDiagLabels, HiddenUnicodeCodepointsDiagSub,
-};
-use crate::{EarlyContext, EarlyLintPass, LintContext};
-
-declare_lint! {
-    #[allow(text_direction_codepoint_in_literal)]
-    /// The `text_direction_codepoint_in_literal` lint detects Unicode codepoints that change the
-    /// visual representation of text on screen in a way that does not correspond to their on
-    /// memory representation.
-    ///
-    /// ### Explanation
-    ///
-    /// The unicode characters `\u{202A}`, `\u{202B}`, `\u{202D}`, `\u{202E}`, `\u{2066}`,
-    /// `\u{2067}`, `\u{2068}`, `\u{202C}` and `\u{2069}` make the flow of text on screen change
-    /// its direction on software that supports these codepoints. This makes the text "abc" display
-    /// as "cba" on screen. By leveraging software that supports these, people can write specially
-    /// crafted literals that make the surrounding code seem like it's performing one action, when
-    /// in reality it is performing another. Because of this, we proactively lint against their
-    /// presence to avoid surprises.
-    ///
-    /// ### Example
-    ///
-    /// ```rust,compile_fail
-    /// #![deny(text_direction_codepoint_in_literal)]
-    /// fn main() {
-    ///     println!("{:?}", '‮');
-    /// }
-    /// ```
-    ///
-    /// {{produces}}
-    ///
-    pub TEXT_DIRECTION_CODEPOINT_IN_LITERAL,
-    Deny,
-    "detect special Unicode codepoints that affect the visual representation of text on screen, \
-     changing the direction in which text flows",
-}
-
-declare_lint_pass!(HiddenUnicodeCodepoints => [TEXT_DIRECTION_CODEPOINT_IN_LITERAL]);
-
-impl HiddenUnicodeCodepoints {
-    fn lint_text_direction_codepoint(
-        &self,
-        cx: &EarlyContext<'_>,
-        text: Symbol,
-        span: Span,
-        padding: u32,
-        point_at_inner_spans: bool,
-        label: &str,
-    ) {
-        // Obtain the `Span`s for each of the forbidden chars.
-        let spans: Vec<_> = text
-            .as_str()
-            .char_indices()
-            .filter_map(|(i, c)| {
-                TEXT_FLOW_CONTROL_CHARS.contains(&c).then(|| {
-                    let lo = span.lo() + BytePos(i as u32 + padding);
-                    (c, span.with_lo(lo).with_hi(lo + BytePos(c.len_utf8() as u32)))
-                })
-            })
-            .collect();
-
-        let count = spans.len();
-        let labels = point_at_inner_spans
-            .then_some(HiddenUnicodeCodepointsDiagLabels { spans: spans.clone() });
-        let sub = if point_at_inner_spans && !spans.is_empty() {
-            HiddenUnicodeCodepointsDiagSub::Escape { spans }
-        } else {
-            HiddenUnicodeCodepointsDiagSub::NoEscape { spans }
-        };
-
-        cx.emit_span_lint(
-            TEXT_DIRECTION_CODEPOINT_IN_LITERAL,
-            span,
-            HiddenUnicodeCodepointsDiag { label, count, span_label: span, labels, sub },
-        );
-    }
-
-    fn check_literal(
-        &mut self,
-        cx: &EarlyContext<'_>,
-        text: Symbol,
-        lit_kind: ast::token::LitKind,
-        span: Span,
-        label: &'static str,
-    ) {
-        if !contains_text_flow_control_chars(text.as_str()) {
-            return;
-        }
-        let (padding, point_at_inner_spans) = match lit_kind {
-            // account for `"` or `'`
-            ast::token::LitKind::Str | ast::token::LitKind::Char => (1, true),
-            // account for `c"`
-            ast::token::LitKind::CStr => (2, true),
-            // account for `r###"`
-            ast::token::LitKind::StrRaw(n) => (n as u32 + 2, true),
-            // account for `cr###"`
-            ast::token::LitKind::CStrRaw(n) => (n as u32 + 3, true),
-            // suppress bad literals.
-            ast::token::LitKind::Err(_) => return,
-            // Be conservative just in case new literals do support these.
-            _ => (0, false),
-        };
-        self.lint_text_direction_codepoint(cx, text, span, padding, point_at_inner_spans, label);
-    }
-}
-
-impl EarlyLintPass for HiddenUnicodeCodepoints {
-    fn check_attribute(&mut self, cx: &EarlyContext<'_>, attr: &ast::Attribute) {
-        if let ast::AttrKind::DocComment(_, comment) = attr.kind {
-            if contains_text_flow_control_chars(comment.as_str()) {
-                self.lint_text_direction_codepoint(cx, comment, attr.span, 0, false, "doc comment");
-            }
-        }
-    }
-
-    #[inline]
-    fn check_expr(&mut self, cx: &EarlyContext<'_>, expr: &ast::Expr) {
-        // byte strings are already handled well enough by `EscapeError::NonAsciiCharInByteString`
-        match &expr.kind {
-            ast::ExprKind::Lit(token_lit) => {
-                self.check_literal(cx, token_lit.symbol, token_lit.kind, expr.span, "literal");
-            }
-            ast::ExprKind::FormatArgs(args) => {
-                let (lit_kind, text) = args.uncooked_fmt_str;
-                self.check_literal(cx, text, lit_kind, args.span, "format string");
-            }
-            _ => {}
-        };
-    }
-}
diff --git a/compiler/rustc_lint/src/impl_trait_overcaptures.rs b/compiler/rustc_lint/src/impl_trait_overcaptures.rs
index a8f45d043be..8124d7f7c86 100644
--- a/compiler/rustc_lint/src/impl_trait_overcaptures.rs
+++ b/compiler/rustc_lint/src/impl_trait_overcaptures.rs
@@ -461,7 +461,7 @@ fn extract_def_id_from_arg<'tcx>(
     generics: &'tcx ty::Generics,
     arg: ty::GenericArg<'tcx>,
 ) -> DefId {
-    match arg.unpack() {
+    match arg.kind() {
         ty::GenericArgKind::Lifetime(re) => match re.kind() {
             ty::ReEarlyParam(ebr) => generics.region_param(ebr, tcx).def_id,
             ty::ReBound(
diff --git a/compiler/rustc_lint/src/internal.rs b/compiler/rustc_lint/src/internal.rs
index 1d4be24ea9f..1805a674d68 100644
--- a/compiler/rustc_lint/src/internal.rs
+++ b/compiler/rustc_lint/src/internal.rs
@@ -328,16 +328,19 @@ impl<'tcx> LateLintPass<'tcx> for TypeIr {
     fn check_item(&mut self, cx: &LateContext<'tcx>, item: &'tcx hir::Item<'tcx>) {
         let rustc_hir::ItemKind::Use(path, kind) = item.kind else { return };
 
-        let is_mod_inherent = |def_id| cx.tcx.is_diagnostic_item(sym::type_ir_inherent, def_id);
+        let is_mod_inherent = |res: Res| {
+            res.opt_def_id()
+                .is_some_and(|def_id| cx.tcx.is_diagnostic_item(sym::type_ir_inherent, def_id))
+        };
 
         // Path segments except for the final.
-        if let Some(seg) =
-            path.segments.iter().find(|seg| seg.res.opt_def_id().is_some_and(is_mod_inherent))
-        {
+        if let Some(seg) = path.segments.iter().find(|seg| is_mod_inherent(seg.res)) {
             cx.emit_span_lint(USAGE_OF_TYPE_IR_INHERENT, seg.ident.span, TypeIrInherentUsage);
         }
         // Final path resolutions, like `use rustc_type_ir::inherent`
-        else if path.res.iter().any(|res| res.opt_def_id().is_some_and(is_mod_inherent)) {
+        else if let Some(type_ns) = path.res.type_ns
+            && is_mod_inherent(type_ns)
+        {
             cx.emit_span_lint(
                 USAGE_OF_TYPE_IR_INHERENT,
                 path.segments.last().unwrap().ident.span,
@@ -346,13 +349,12 @@ impl<'tcx> LateLintPass<'tcx> for TypeIr {
         }
 
         let (lo, hi, snippet) = match path.segments {
-            [.., penultimate, segment]
-                if penultimate.res.opt_def_id().is_some_and(is_mod_inherent) =>
-            {
+            [.., penultimate, segment] if is_mod_inherent(penultimate.res) => {
                 (segment.ident.span, item.kind.ident().unwrap().span, "*")
             }
             [.., segment]
-                if path.res.iter().flat_map(Res::opt_def_id).any(is_mod_inherent)
+                if let Some(type_ns) = path.res.type_ns
+                    && is_mod_inherent(type_ns)
                     && let rustc_hir::UseKind::Single(ident) = kind =>
             {
                 let (lo, snippet) =
diff --git a/compiler/rustc_lint/src/levels.rs b/compiler/rustc_lint/src/levels.rs
index 00775647ac6..c52dbd892bf 100644
--- a/compiler/rustc_lint/src/levels.rs
+++ b/compiler/rustc_lint/src/levels.rs
@@ -126,7 +126,7 @@ fn lints_that_dont_need_to_run(tcx: TyCtxt<'_>, (): ()) -> UnordSet<LintId> {
         .filter(|lint| {
             // Lints that show up in future-compat reports must always be run.
             let has_future_breakage =
-                lint.future_incompatible.is_some_and(|fut| fut.reason.has_future_breakage());
+                lint.future_incompatible.is_some_and(|fut| fut.report_in_deps);
             !has_future_breakage && !lint.eval_always
         })
         .filter(|lint| {
diff --git a/compiler/rustc_lint/src/lib.rs b/compiler/rustc_lint/src/lib.rs
index 4ff586a79a6..72bfeaddbf1 100644
--- a/compiler/rustc_lint/src/lib.rs
+++ b/compiler/rustc_lint/src/lib.rs
@@ -48,7 +48,6 @@ mod errors;
 mod expect;
 mod for_loops_over_fallibles;
 mod foreign_modules;
-pub mod hidden_unicode_codepoints;
 mod if_let_rescope;
 mod impl_trait_overcaptures;
 mod internal;
@@ -56,6 +55,7 @@ mod invalid_from_utf8;
 mod late;
 mod let_underscore;
 mod levels;
+mod lifetime_syntax;
 mod lints;
 mod macro_expr_fragment_specifier_2024_migration;
 mod map_unit_fn;
@@ -75,6 +75,7 @@ mod reference_casting;
 mod shadowed_into_iter;
 mod static_mut_refs;
 mod traits;
+mod transmute;
 mod types;
 mod unit_bindings;
 mod unqualified_local_imports;
@@ -91,12 +92,12 @@ use deref_into_dyn_supertrait::*;
 use drop_forget_useless::*;
 use enum_intrinsics_non_enums::EnumIntrinsicsNonEnums;
 use for_loops_over_fallibles::*;
-use hidden_unicode_codepoints::*;
 use if_let_rescope::IfLetRescope;
 use impl_trait_overcaptures::ImplTraitOvercaptures;
 use internal::*;
 use invalid_from_utf8::*;
 use let_underscore::*;
+use lifetime_syntax::*;
 use macro_expr_fragment_specifier_2024_migration::*;
 use map_unit_fn::*;
 use multiple_supertrait_upcastable::*;
@@ -118,6 +119,7 @@ use shadowed_into_iter::ShadowedIntoIter;
 pub use shadowed_into_iter::{ARRAY_INTO_ITER, BOXED_SLICE_INTO_ITER};
 use static_mut_refs::*;
 use traits::*;
+use transmute::CheckTransmutes;
 use types::*;
 use unit_bindings::*;
 use unqualified_local_imports::*;
@@ -126,6 +128,7 @@ use unused::*;
 #[rustfmt::skip]
 pub use builtin::{MissingDoc, SoftLints};
 pub use context::{CheckLintNameResult, EarlyContext, LateContext, LintContext, LintStore};
+pub use early::diagnostics::decorate_builtin_lint;
 pub use early::{EarlyCheckNode, check_ast_node};
 pub use late::{check_crate, late_lint_mod, unerased_lint_store};
 pub use levels::LintLevelsBuilder;
@@ -174,7 +177,6 @@ early_lint_methods!(
             DeprecatedAttr: DeprecatedAttr::default(),
             WhileTrue: WhileTrue,
             NonAsciiIdents: NonAsciiIdents,
-            HiddenUnicodeCodepoints: HiddenUnicodeCodepoints,
             IncompleteInternalFeatures: IncompleteInternalFeatures,
             RedundantSemicolons: RedundantSemicolons,
             UnusedDocComment: UnusedDocComment,
@@ -245,6 +247,8 @@ late_lint_methods!(
             IfLetRescope: IfLetRescope::default(),
             StaticMutRefs: StaticMutRefs,
             UnqualifiedLocalImports: UnqualifiedLocalImports,
+            CheckTransmutes: CheckTransmutes,
+            LifetimeSyntax: LifetimeSyntax,
         ]
     ]
 );
@@ -352,6 +356,7 @@ fn register_builtins(store: &mut LintStore) {
     store.register_renamed("unused_tuple_struct_fields", "dead_code");
     store.register_renamed("static_mut_ref", "static_mut_refs");
     store.register_renamed("temporary_cstring_as_ptr", "dangling_pointers_from_temporaries");
+    store.register_renamed("elided_named_lifetimes", "mismatched_lifetime_syntaxes");
 
     // These were moved to tool lints, but rustc still sees them when compiling normally, before
     // tool lints are registered, so `check_tool_name_for_backwards_compat` doesn't work. Use
@@ -594,7 +599,6 @@ fn register_builtins(store: &mut LintStore) {
         "converted into hard error, see PR #125380 \
          <https://github.com/rust-lang/rust/pull/125380> for more information",
     );
-    store.register_removed("unsupported_calling_conventions", "converted into hard error");
     store.register_removed(
         "cenum_impl_drop_cast",
         "converted into hard error, \
diff --git a/compiler/rustc_lint/src/lifetime_syntax.rs b/compiler/rustc_lint/src/lifetime_syntax.rs
new file mode 100644
index 00000000000..31b038e6a46
--- /dev/null
+++ b/compiler/rustc_lint/src/lifetime_syntax.rs
@@ -0,0 +1,503 @@
+use rustc_data_structures::fx::FxIndexMap;
+use rustc_hir::intravisit::{self, Visitor};
+use rustc_hir::{self as hir, LifetimeSource};
+use rustc_session::{declare_lint, declare_lint_pass};
+use rustc_span::Span;
+use tracing::instrument;
+
+use crate::{LateContext, LateLintPass, LintContext, lints};
+
+declare_lint! {
+    /// The `mismatched_lifetime_syntaxes` lint detects when the same
+    /// lifetime is referred to by different syntaxes between function
+    /// arguments and return values.
+    ///
+    /// The three kinds of syntaxes are:
+    ///
+    /// 1. Named lifetimes. These are references (`&'a str`) or paths
+    ///    (`Person<'a>`) that use a lifetime with a name, such as
+    ///    `'static` or `'a`.
+    ///
+    /// 2. Elided lifetimes. These are references with no explicit
+    ///    lifetime (`&str`), references using the anonymous lifetime
+    ///    (`&'_ str`), and paths using the anonymous lifetime
+    ///    (`Person<'_>`).
+    ///
+    /// 3. Hidden lifetimes. These are paths that do not contain any
+    ///    visual indication that it contains a lifetime (`Person`).
+    ///
+    /// ### Example
+    ///
+    /// ```rust,compile_fail
+    /// #![deny(mismatched_lifetime_syntaxes)]
+    ///
+    /// pub fn mixing_named_with_elided(v: &'static u8) -> &u8 {
+    ///     v
+    /// }
+    ///
+    /// struct Person<'a> {
+    ///     name: &'a str,
+    /// }
+    ///
+    /// pub fn mixing_hidden_with_elided(v: Person) -> Person<'_> {
+    ///     v
+    /// }
+    ///
+    /// struct Foo;
+    ///
+    /// impl Foo {
+    ///     // Lifetime elision results in the output lifetime becoming
+    ///     // `'static`, which is not what was intended.
+    ///     pub fn get_mut(&'static self, x: &mut u8) -> &mut u8 {
+    ///         unsafe { &mut *(x as *mut _) }
+    ///     }
+    /// }
+    /// ```
+    ///
+    /// {{produces}}
+    ///
+    /// ### Explanation
+    ///
+    /// Lifetime elision is useful because it frees you from having to
+    /// give each lifetime its own name and show the relation of input
+    /// and output lifetimes for common cases. However, a lifetime
+    /// that uses inconsistent syntax between related arguments and
+    /// return values is more confusing.
+    ///
+    /// In certain `unsafe` code, lifetime elision combined with
+    /// inconsistent lifetime syntax may result in unsound code.
+    pub MISMATCHED_LIFETIME_SYNTAXES,
+    Warn,
+    "detects when a lifetime uses different syntax between arguments and return values"
+}
+
+declare_lint_pass!(LifetimeSyntax => [MISMATCHED_LIFETIME_SYNTAXES]);
+
+impl<'tcx> LateLintPass<'tcx> for LifetimeSyntax {
+    #[instrument(skip_all)]
+    fn check_fn(
+        &mut self,
+        cx: &LateContext<'tcx>,
+        _: hir::intravisit::FnKind<'tcx>,
+        fd: &'tcx hir::FnDecl<'tcx>,
+        _: &'tcx hir::Body<'tcx>,
+        _: rustc_span::Span,
+        _: rustc_span::def_id::LocalDefId,
+    ) {
+        let mut input_map = Default::default();
+        let mut output_map = Default::default();
+
+        for input in fd.inputs {
+            LifetimeInfoCollector::collect(input, &mut input_map);
+        }
+
+        if let hir::FnRetTy::Return(output) = fd.output {
+            LifetimeInfoCollector::collect(output, &mut output_map);
+        }
+
+        report_mismatches(cx, &input_map, &output_map);
+    }
+}
+
+#[instrument(skip_all)]
+fn report_mismatches<'tcx>(
+    cx: &LateContext<'tcx>,
+    inputs: &LifetimeInfoMap<'tcx>,
+    outputs: &LifetimeInfoMap<'tcx>,
+) {
+    for (resolved_lifetime, output_info) in outputs {
+        if let Some(input_info) = inputs.get(resolved_lifetime) {
+            if !lifetimes_use_matched_syntax(input_info, output_info) {
+                emit_mismatch_diagnostic(cx, input_info, output_info);
+            }
+        }
+    }
+}
+
+fn lifetimes_use_matched_syntax(input_info: &[Info<'_>], output_info: &[Info<'_>]) -> bool {
+    // Categorize lifetimes into source/syntax buckets.
+    let mut n_hidden = 0;
+    let mut n_elided = 0;
+    let mut n_named = 0;
+
+    for info in input_info.iter().chain(output_info) {
+        use LifetimeSource::*;
+        use hir::LifetimeSyntax::*;
+
+        let syntax_source = (info.lifetime.syntax, info.lifetime.source);
+
+        match syntax_source {
+            // Ignore any other kind of lifetime.
+            (_, Other) => continue,
+
+            // E.g. `&T`.
+            (Implicit, Reference | OutlivesBound | PreciseCapturing) |
+            // E.g. `&'_ T`.
+            (ExplicitAnonymous, Reference | OutlivesBound | PreciseCapturing) |
+            // E.g. `ContainsLifetime<'_>`.
+            (ExplicitAnonymous, Path { .. }) => n_elided += 1,
+
+            // E.g. `ContainsLifetime`.
+            (Implicit, Path { .. }) => n_hidden += 1,
+
+            // E.g. `&'a T`.
+            (ExplicitBound, Reference | OutlivesBound | PreciseCapturing) |
+            // E.g. `ContainsLifetime<'a>`.
+            (ExplicitBound, Path { .. }) => n_named += 1,
+        };
+    }
+
+    let syntax_counts = (n_hidden, n_elided, n_named);
+    tracing::debug!(?syntax_counts);
+
+    matches!(syntax_counts, (_, 0, 0) | (0, _, 0) | (0, 0, _))
+}
+
+fn emit_mismatch_diagnostic<'tcx>(
+    cx: &LateContext<'tcx>,
+    input_info: &[Info<'_>],
+    output_info: &[Info<'_>],
+) {
+    // There can only ever be zero or one bound lifetime
+    // for a given lifetime resolution.
+    let mut bound_lifetime = None;
+
+    // We offer the following kinds of suggestions (when appropriate
+    // such that the suggestion wouldn't violate the lint):
+    //
+    // 1. Every lifetime becomes named, when there is already a
+    //    user-provided name.
+    //
+    // 2. A "mixed" signature, where references become implicit
+    //    and paths become explicitly anonymous.
+    //
+    // 3. Every lifetime becomes implicit.
+    //
+    // 4. Every lifetime becomes explicitly anonymous.
+    //
+    // Number 2 is arguably the most common pattern and the one we
+    // should push strongest. Number 3 is likely the next most common,
+    // followed by number 1. Coming in at a distant last would be
+    // number 4.
+    //
+    // Beyond these, there are variants of acceptable signatures that
+    // we won't suggest because they are very low-value. For example,
+    // we will never suggest `fn(&T1, &'_ T2) -> &T3` even though that
+    // would pass the lint.
+    //
+    // The following collections are the lifetime instances that we
+    // suggest changing to a given alternate style.
+
+    // 1. Convert all to named.
+    let mut suggest_change_to_explicit_bound = Vec::new();
+
+    // 2. Convert to mixed. We track each kind of change separately.
+    let mut suggest_change_to_mixed_implicit = Vec::new();
+    let mut suggest_change_to_mixed_explicit_anonymous = Vec::new();
+
+    // 3. Convert all to implicit.
+    let mut suggest_change_to_implicit = Vec::new();
+
+    // 4. Convert all to explicit anonymous.
+    let mut suggest_change_to_explicit_anonymous = Vec::new();
+
+    // Some styles prevent using implicit syntax at all.
+    let mut allow_suggesting_implicit = true;
+
+    // It only makes sense to suggest mixed if we have both sources.
+    let mut saw_a_reference = false;
+    let mut saw_a_path = false;
+
+    for info in input_info.iter().chain(output_info) {
+        use LifetimeSource::*;
+        use hir::LifetimeSyntax::*;
+
+        let syntax_source = (info.lifetime.syntax, info.lifetime.source);
+
+        if let (_, Other) = syntax_source {
+            // Ignore any other kind of lifetime.
+            continue;
+        }
+
+        if let (ExplicitBound, _) = syntax_source {
+            bound_lifetime = Some(info);
+        }
+
+        match syntax_source {
+            // E.g. `&T`.
+            (Implicit, Reference) => {
+                suggest_change_to_explicit_anonymous.push(info);
+                suggest_change_to_explicit_bound.push(info);
+            }
+
+            // E.g. `&'_ T`.
+            (ExplicitAnonymous, Reference) => {
+                suggest_change_to_implicit.push(info);
+                suggest_change_to_mixed_implicit.push(info);
+                suggest_change_to_explicit_bound.push(info);
+            }
+
+            // E.g. `ContainsLifetime`.
+            (Implicit, Path { .. }) => {
+                suggest_change_to_mixed_explicit_anonymous.push(info);
+                suggest_change_to_explicit_anonymous.push(info);
+                suggest_change_to_explicit_bound.push(info);
+            }
+
+            // E.g. `ContainsLifetime<'_>`.
+            (ExplicitAnonymous, Path { .. }) => {
+                suggest_change_to_explicit_bound.push(info);
+            }
+
+            // E.g. `&'a T`.
+            (ExplicitBound, Reference) => {
+                suggest_change_to_implicit.push(info);
+                suggest_change_to_mixed_implicit.push(info);
+                suggest_change_to_explicit_anonymous.push(info);
+            }
+
+            // E.g. `ContainsLifetime<'a>`.
+            (ExplicitBound, Path { .. }) => {
+                suggest_change_to_mixed_explicit_anonymous.push(info);
+                suggest_change_to_explicit_anonymous.push(info);
+            }
+
+            (Implicit, OutlivesBound | PreciseCapturing) => {
+                panic!("This syntax / source combination is not possible");
+            }
+
+            // E.g. `+ '_`, `+ use<'_>`.
+            (ExplicitAnonymous, OutlivesBound | PreciseCapturing) => {
+                suggest_change_to_explicit_bound.push(info);
+            }
+
+            // E.g. `+ 'a`, `+ use<'a>`.
+            (ExplicitBound, OutlivesBound | PreciseCapturing) => {
+                suggest_change_to_mixed_explicit_anonymous.push(info);
+                suggest_change_to_explicit_anonymous.push(info);
+            }
+
+            (_, Other) => {
+                panic!("This syntax / source combination has already been skipped");
+            }
+        }
+
+        if matches!(syntax_source, (_, Path { .. } | OutlivesBound | PreciseCapturing)) {
+            allow_suggesting_implicit = false;
+        }
+
+        match syntax_source {
+            (_, Reference) => saw_a_reference = true,
+            (_, Path { .. }) => saw_a_path = true,
+            _ => {}
+        }
+    }
+
+    let make_implicit_suggestions =
+        |infos: &[&Info<'_>]| infos.iter().map(|i| i.removing_span()).collect::<Vec<_>>();
+
+    let inputs = input_info.iter().map(|info| info.reporting_span()).collect();
+    let outputs = output_info.iter().map(|info| info.reporting_span()).collect();
+
+    let explicit_bound_suggestion = bound_lifetime.map(|info| {
+        build_mismatch_suggestion(info.lifetime_name(), &suggest_change_to_explicit_bound)
+    });
+
+    let is_bound_static = bound_lifetime.is_some_and(|info| info.is_static());
+
+    tracing::debug!(?bound_lifetime, ?explicit_bound_suggestion, ?is_bound_static);
+
+    let should_suggest_mixed =
+        // Do we have a mixed case?
+        (saw_a_reference && saw_a_path) &&
+        // Is there anything to change?
+        (!suggest_change_to_mixed_implicit.is_empty() ||
+         !suggest_change_to_mixed_explicit_anonymous.is_empty()) &&
+        // If we have `'static`, we don't want to remove it.
+        !is_bound_static;
+
+    let mixed_suggestion = should_suggest_mixed.then(|| {
+        let implicit_suggestions = make_implicit_suggestions(&suggest_change_to_mixed_implicit);
+
+        let explicit_anonymous_suggestions = suggest_change_to_mixed_explicit_anonymous
+            .iter()
+            .map(|info| info.suggestion("'_"))
+            .collect();
+
+        lints::MismatchedLifetimeSyntaxesSuggestion::Mixed {
+            implicit_suggestions,
+            explicit_anonymous_suggestions,
+            tool_only: false,
+        }
+    });
+
+    tracing::debug!(
+        ?suggest_change_to_mixed_implicit,
+        ?suggest_change_to_mixed_explicit_anonymous,
+        ?mixed_suggestion,
+    );
+
+    let should_suggest_implicit =
+        // Is there anything to change?
+        !suggest_change_to_implicit.is_empty() &&
+        // We never want to hide the lifetime in a path (or similar).
+        allow_suggesting_implicit &&
+        // If we have `'static`, we don't want to remove it.
+        !is_bound_static;
+
+    let implicit_suggestion = should_suggest_implicit.then(|| {
+        let suggestions = make_implicit_suggestions(&suggest_change_to_implicit);
+
+        lints::MismatchedLifetimeSyntaxesSuggestion::Implicit { suggestions, tool_only: false }
+    });
+
+    tracing::debug!(
+        ?should_suggest_implicit,
+        ?suggest_change_to_implicit,
+        allow_suggesting_implicit,
+        ?implicit_suggestion,
+    );
+
+    let should_suggest_explicit_anonymous =
+        // Is there anything to change?
+        !suggest_change_to_explicit_anonymous.is_empty() &&
+        // If we have `'static`, we don't want to remove it.
+        !is_bound_static;
+
+    let explicit_anonymous_suggestion = should_suggest_explicit_anonymous
+        .then(|| build_mismatch_suggestion("'_", &suggest_change_to_explicit_anonymous));
+
+    tracing::debug!(
+        ?should_suggest_explicit_anonymous,
+        ?suggest_change_to_explicit_anonymous,
+        ?explicit_anonymous_suggestion,
+    );
+
+    let lifetime_name = bound_lifetime.map(|info| info.lifetime_name()).unwrap_or("'_").to_owned();
+
+    // We can produce a number of suggestions which may overwhelm
+    // the user. Instead, we order the suggestions based on Rust
+    // idioms. The "best" choice is shown to the user and the
+    // remaining choices are shown to tools only.
+    let mut suggestions = Vec::new();
+    suggestions.extend(explicit_bound_suggestion);
+    suggestions.extend(mixed_suggestion);
+    suggestions.extend(implicit_suggestion);
+    suggestions.extend(explicit_anonymous_suggestion);
+
+    cx.emit_span_lint(
+        MISMATCHED_LIFETIME_SYNTAXES,
+        Vec::clone(&inputs),
+        lints::MismatchedLifetimeSyntaxes { lifetime_name, inputs, outputs, suggestions },
+    );
+}
+
+fn build_mismatch_suggestion(
+    lifetime_name: &str,
+    infos: &[&Info<'_>],
+) -> lints::MismatchedLifetimeSyntaxesSuggestion {
+    let lifetime_name = lifetime_name.to_owned();
+
+    let suggestions = infos.iter().map(|info| info.suggestion(&lifetime_name)).collect();
+
+    lints::MismatchedLifetimeSyntaxesSuggestion::Explicit {
+        lifetime_name,
+        suggestions,
+        tool_only: false,
+    }
+}
+
+#[derive(Debug)]
+struct Info<'tcx> {
+    type_span: Span,
+    referenced_type_span: Option<Span>,
+    lifetime: &'tcx hir::Lifetime,
+}
+
+impl<'tcx> Info<'tcx> {
+    fn lifetime_name(&self) -> &str {
+        self.lifetime.ident.as_str()
+    }
+
+    fn is_static(&self) -> bool {
+        self.lifetime.is_static()
+    }
+
+    /// When reporting a lifetime that is implicit, we expand the span
+    /// to include the type. Otherwise we end up pointing at nothing,
+    /// which is a bit confusing.
+    fn reporting_span(&self) -> Span {
+        if self.lifetime.is_implicit() { self.type_span } else { self.lifetime.ident.span }
+    }
+
+    /// When removing an explicit lifetime from a reference,
+    /// we want to remove the whitespace after the lifetime.
+    ///
+    /// ```rust
+    /// fn x(a: &'_ u8) {}
+    /// ```
+    ///
+    /// Should become:
+    ///
+    /// ```rust
+    /// fn x(a: &u8) {}
+    /// ```
+    // FIXME: Ideally, we'd also remove the lifetime declaration.
+    fn removing_span(&self) -> Span {
+        let mut span = self.suggestion("'dummy").0;
+
+        if let Some(referenced_type_span) = self.referenced_type_span {
+            span = span.until(referenced_type_span);
+        }
+
+        span
+    }
+
+    fn suggestion(&self, lifetime_name: &str) -> (Span, String) {
+        self.lifetime.suggestion(lifetime_name)
+    }
+}
+
+type LifetimeInfoMap<'tcx> = FxIndexMap<&'tcx hir::LifetimeKind, Vec<Info<'tcx>>>;
+
+struct LifetimeInfoCollector<'a, 'tcx> {
+    type_span: Span,
+    referenced_type_span: Option<Span>,
+    map: &'a mut LifetimeInfoMap<'tcx>,
+}
+
+impl<'a, 'tcx> LifetimeInfoCollector<'a, 'tcx> {
+    fn collect(ty: &'tcx hir::Ty<'tcx>, map: &'a mut LifetimeInfoMap<'tcx>) {
+        let mut this = Self { type_span: ty.span, referenced_type_span: None, map };
+
+        intravisit::walk_unambig_ty(&mut this, ty);
+    }
+}
+
+impl<'a, 'tcx> Visitor<'tcx> for LifetimeInfoCollector<'a, 'tcx> {
+    #[instrument(skip(self))]
+    fn visit_lifetime(&mut self, lifetime: &'tcx hir::Lifetime) {
+        let type_span = self.type_span;
+        let referenced_type_span = self.referenced_type_span;
+
+        let info = Info { type_span, referenced_type_span, lifetime };
+
+        self.map.entry(&lifetime.kind).or_default().push(info);
+    }
+
+    #[instrument(skip(self))]
+    fn visit_ty(&mut self, ty: &'tcx hir::Ty<'tcx, hir::AmbigArg>) -> Self::Result {
+        let old_type_span = self.type_span;
+        let old_referenced_type_span = self.referenced_type_span;
+
+        self.type_span = ty.span;
+        if let hir::TyKind::Ref(_, ty) = ty.kind {
+            self.referenced_type_span = Some(ty.ty.span);
+        }
+
+        intravisit::walk_ty(self, ty);
+
+        self.type_span = old_type_span;
+        self.referenced_type_span = old_referenced_type_span;
+    }
+}
diff --git a/compiler/rustc_lint/src/lints.rs b/compiler/rustc_lint/src/lints.rs
index 7268a7f704f..9d3c74a9a2b 100644
--- a/compiler/rustc_lint/src/lints.rs
+++ b/compiler/rustc_lint/src/lints.rs
@@ -8,17 +8,17 @@ use rustc_errors::{
     Applicability, Diag, DiagArgValue, DiagMessage, DiagStyledString, ElidedLifetimeInPathSubdiag,
     EmissionGuarantee, LintDiagnostic, MultiSpan, Subdiagnostic, SuggestionStyle,
 };
+use rustc_hir as hir;
 use rustc_hir::def::Namespace;
 use rustc_hir::def_id::DefId;
 use rustc_hir::intravisit::VisitorExt;
-use rustc_hir::{self as hir, MissingLifetimeKind};
 use rustc_macros::{LintDiagnostic, Subdiagnostic};
 use rustc_middle::ty::inhabitedness::InhabitedPredicate;
 use rustc_middle::ty::{Clause, PolyExistentialTraitRef, Ty, TyCtxt};
 use rustc_session::Session;
 use rustc_session::lint::AmbiguityErrorDiag;
 use rustc_span::edition::Edition;
-use rustc_span::{Ident, MacroRulesNormalizedIdent, Span, Symbol, kw, sym};
+use rustc_span::{Ident, MacroRulesNormalizedIdent, Span, Symbol, sym};
 
 use crate::builtin::{InitError, ShorthandAssocTyCollector, TypeAliasBounds};
 use crate::errors::{OverruledAttributeSub, RequestedLevel};
@@ -1782,13 +1782,20 @@ pub(crate) enum InvalidNanComparisonsSuggestion {
 #[derive(LintDiagnostic)]
 pub(crate) enum AmbiguousWidePointerComparisons<'a> {
     #[diag(lint_ambiguous_wide_pointer_comparisons)]
-    Spanful {
+    SpanfulEq {
         #[subdiagnostic]
         addr_suggestion: AmbiguousWidePointerComparisonsAddrSuggestion<'a>,
         #[subdiagnostic]
         addr_metadata_suggestion: Option<AmbiguousWidePointerComparisonsAddrMetadataSuggestion<'a>>,
     },
     #[diag(lint_ambiguous_wide_pointer_comparisons)]
+    SpanfulCmp {
+        #[subdiagnostic]
+        cast_suggestion: AmbiguousWidePointerComparisonsCastSuggestion<'a>,
+        #[subdiagnostic]
+        expect_suggestion: AmbiguousWidePointerComparisonsExpectSuggestion<'a>,
+    },
+    #[diag(lint_ambiguous_wide_pointer_comparisons)]
     #[help(lint_addr_metadata_suggestion)]
     #[help(lint_addr_suggestion)]
     Spanless,
@@ -1816,48 +1823,67 @@ pub(crate) struct AmbiguousWidePointerComparisonsAddrMetadataSuggestion<'a> {
 }
 
 #[derive(Subdiagnostic)]
-pub(crate) enum AmbiguousWidePointerComparisonsAddrSuggestion<'a> {
-    #[multipart_suggestion(
-        lint_addr_suggestion,
-        style = "verbose",
-        // FIXME(#53934): make machine-applicable again
-        applicability = "maybe-incorrect"
-    )]
-    AddrEq {
-        ne: &'a str,
-        deref_left: &'a str,
-        deref_right: &'a str,
-        l_modifiers: &'a str,
-        r_modifiers: &'a str,
-        #[suggestion_part(code = "{ne}std::ptr::addr_eq({deref_left}")]
-        left: Span,
-        #[suggestion_part(code = "{l_modifiers}, {deref_right}")]
-        middle: Span,
-        #[suggestion_part(code = "{r_modifiers})")]
-        right: Span,
-    },
-    #[multipart_suggestion(
-        lint_addr_suggestion,
-        style = "verbose",
-        // FIXME(#53934): make machine-applicable again
-        applicability = "maybe-incorrect"
+#[multipart_suggestion(
+    lint_addr_suggestion,
+    style = "verbose",
+    // FIXME(#53934): make machine-applicable again
+    applicability = "maybe-incorrect"
+)]
+pub(crate) struct AmbiguousWidePointerComparisonsAddrSuggestion<'a> {
+    pub(crate) ne: &'a str,
+    pub(crate) deref_left: &'a str,
+    pub(crate) deref_right: &'a str,
+    pub(crate) l_modifiers: &'a str,
+    pub(crate) r_modifiers: &'a str,
+    #[suggestion_part(code = "{ne}std::ptr::addr_eq({deref_left}")]
+    pub(crate) left: Span,
+    #[suggestion_part(code = "{l_modifiers}, {deref_right}")]
+    pub(crate) middle: Span,
+    #[suggestion_part(code = "{r_modifiers})")]
+    pub(crate) right: Span,
+}
+
+#[derive(Subdiagnostic)]
+#[multipart_suggestion(
+    lint_cast_suggestion,
+    style = "verbose",
+    // FIXME(#53934): make machine-applicable again
+    applicability = "maybe-incorrect"
+)]
+pub(crate) struct AmbiguousWidePointerComparisonsCastSuggestion<'a> {
+    pub(crate) deref_left: &'a str,
+    pub(crate) deref_right: &'a str,
+    pub(crate) paren_left: &'a str,
+    pub(crate) paren_right: &'a str,
+    pub(crate) l_modifiers: &'a str,
+    pub(crate) r_modifiers: &'a str,
+    #[suggestion_part(code = "({deref_left}")]
+    pub(crate) left_before: Option<Span>,
+    #[suggestion_part(code = "{l_modifiers}{paren_left}.cast::<()>()")]
+    pub(crate) left_after: Span,
+    #[suggestion_part(code = "({deref_right}")]
+    pub(crate) right_before: Option<Span>,
+    #[suggestion_part(code = "{r_modifiers}{paren_right}.cast::<()>()")]
+    pub(crate) right_after: Span,
+}
+
+#[derive(Subdiagnostic)]
+#[multipart_suggestion(
+    lint_expect_suggestion,
+    style = "verbose",
+    // FIXME(#53934): make machine-applicable again
+    applicability = "maybe-incorrect"
+)]
+pub(crate) struct AmbiguousWidePointerComparisonsExpectSuggestion<'a> {
+    pub(crate) paren_left: &'a str,
+    pub(crate) paren_right: &'a str,
+    // FIXME(#127436): Adjust once resolved
+    #[suggestion_part(
+        code = r#"{{ #[expect(ambiguous_wide_pointer_comparisons, reason = "...")] {paren_left}"#
     )]
-    Cast {
-        deref_left: &'a str,
-        deref_right: &'a str,
-        paren_left: &'a str,
-        paren_right: &'a str,
-        l_modifiers: &'a str,
-        r_modifiers: &'a str,
-        #[suggestion_part(code = "({deref_left}")]
-        left_before: Option<Span>,
-        #[suggestion_part(code = "{l_modifiers}{paren_left}.cast::<()>()")]
-        left_after: Span,
-        #[suggestion_part(code = "({deref_right}")]
-        right_before: Option<Span>,
-        #[suggestion_part(code = "{r_modifiers}{paren_right}.cast::<()>()")]
-        right_after: Span,
-    },
+    pub(crate) before: Span,
+    #[suggestion_part(code = "{paren_right} }}")]
+    pub(crate) after: Span,
 }
 
 #[derive(LintDiagnostic)]
@@ -2273,6 +2299,16 @@ pub(crate) mod unexpected_cfg_name {
     pub(crate) enum CodeSuggestion {
         #[help(lint_unexpected_cfg_define_features)]
         DefineFeatures,
+        #[multipart_suggestion(
+            lint_unexpected_cfg_name_version_syntax,
+            applicability = "machine-applicable"
+        )]
+        VersionSyntax {
+            #[suggestion_part(code = "(")]
+            between_name_and_value: Span,
+            #[suggestion_part(code = ")")]
+            after_value: Span,
+        },
         #[suggestion(
             lint_unexpected_cfg_name_similar_name_value,
             applicability = "maybe-incorrect",
@@ -2716,58 +2752,6 @@ pub(crate) struct ElidedLifetimesInPaths {
     pub subdiag: ElidedLifetimeInPathSubdiag,
 }
 
-pub(crate) struct ElidedNamedLifetime {
-    pub span: Span,
-    pub kind: MissingLifetimeKind,
-    pub name: Symbol,
-    pub declaration: Option<Span>,
-}
-
-impl<G: EmissionGuarantee> LintDiagnostic<'_, G> for ElidedNamedLifetime {
-    fn decorate_lint(self, diag: &mut rustc_errors::Diag<'_, G>) {
-        let Self { span, kind, name, declaration } = self;
-        diag.primary_message(fluent::lint_elided_named_lifetime);
-        diag.arg("name", name);
-        diag.span_label(span, fluent::lint_label_elided);
-        if let Some(declaration) = declaration {
-            diag.span_label(declaration, fluent::lint_label_named);
-        }
-        // FIXME(GrigorenkoPV): this `if` and `return` should be removed,
-        //  but currently this lint's suggestions can conflict with those of `clippy::needless_lifetimes`:
-        //  https://github.com/rust-lang/rust/pull/129840#issuecomment-2323349119
-        // HACK: `'static` suggestions will never sonflict, emit only those for now.
-        if name != kw::StaticLifetime {
-            return;
-        }
-        match kind {
-            MissingLifetimeKind::Underscore => diag.span_suggestion_verbose(
-                span,
-                fluent::lint_suggestion,
-                format!("{name}"),
-                Applicability::MachineApplicable,
-            ),
-            MissingLifetimeKind::Ampersand => diag.span_suggestion_verbose(
-                span.shrink_to_hi(),
-                fluent::lint_suggestion,
-                format!("{name} "),
-                Applicability::MachineApplicable,
-            ),
-            MissingLifetimeKind::Comma => diag.span_suggestion_verbose(
-                span.shrink_to_hi(),
-                fluent::lint_suggestion,
-                format!("{name}, "),
-                Applicability::MachineApplicable,
-            ),
-            MissingLifetimeKind::Brackets => diag.span_suggestion_verbose(
-                span.shrink_to_hi(),
-                fluent::lint_suggestion,
-                format!("<{name}>"),
-                Applicability::MachineApplicable,
-            ),
-        };
-    }
-}
-
 #[derive(LintDiagnostic)]
 #[diag(lint_invalid_crate_type_value)]
 pub(crate) struct UnknownCrateTypes {
@@ -3041,7 +3025,9 @@ pub(crate) struct ByteSliceInPackedStructWithDerive {
 #[derive(LintDiagnostic)]
 #[diag(lint_unused_extern_crate)]
 pub(crate) struct UnusedExternCrate {
-    #[suggestion(code = "", applicability = "machine-applicable")]
+    #[label]
+    pub span: Span,
+    #[suggestion(code = "", applicability = "machine-applicable", style = "verbose")]
     pub removal_span: Span,
 }
 
@@ -3203,3 +3189,128 @@ pub(crate) struct ReservedMultihash {
     #[suggestion(code = " ", applicability = "machine-applicable")]
     pub suggestion: Span,
 }
+
+#[derive(Debug)]
+pub(crate) struct MismatchedLifetimeSyntaxes {
+    pub lifetime_name: String,
+    pub inputs: Vec<Span>,
+    pub outputs: Vec<Span>,
+
+    pub suggestions: Vec<MismatchedLifetimeSyntaxesSuggestion>,
+}
+
+impl<'a, G: EmissionGuarantee> LintDiagnostic<'a, G> for MismatchedLifetimeSyntaxes {
+    fn decorate_lint<'b>(self, diag: &'b mut Diag<'a, G>) {
+        diag.primary_message(fluent::lint_mismatched_lifetime_syntaxes);
+
+        diag.arg("lifetime_name", self.lifetime_name);
+
+        diag.arg("n_inputs", self.inputs.len());
+        for input in self.inputs {
+            let a = diag.eagerly_translate(fluent::lint_label_mismatched_lifetime_syntaxes_inputs);
+            diag.span_label(input, a);
+        }
+
+        diag.arg("n_outputs", self.outputs.len());
+        for output in self.outputs {
+            let a = diag.eagerly_translate(fluent::lint_label_mismatched_lifetime_syntaxes_outputs);
+            diag.span_label(output, a);
+        }
+
+        let mut suggestions = self.suggestions.into_iter();
+        if let Some(s) = suggestions.next() {
+            diag.subdiagnostic(s);
+
+            for mut s in suggestions {
+                s.make_tool_only();
+                diag.subdiagnostic(s);
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+pub(crate) enum MismatchedLifetimeSyntaxesSuggestion {
+    Implicit {
+        suggestions: Vec<Span>,
+        tool_only: bool,
+    },
+
+    Mixed {
+        implicit_suggestions: Vec<Span>,
+        explicit_anonymous_suggestions: Vec<(Span, String)>,
+        tool_only: bool,
+    },
+
+    Explicit {
+        lifetime_name: String,
+        suggestions: Vec<(Span, String)>,
+        tool_only: bool,
+    },
+}
+
+impl MismatchedLifetimeSyntaxesSuggestion {
+    fn make_tool_only(&mut self) {
+        use MismatchedLifetimeSyntaxesSuggestion::*;
+
+        let tool_only = match self {
+            Implicit { tool_only, .. } | Mixed { tool_only, .. } | Explicit { tool_only, .. } => {
+                tool_only
+            }
+        };
+
+        *tool_only = true;
+    }
+}
+
+impl Subdiagnostic for MismatchedLifetimeSyntaxesSuggestion {
+    fn add_to_diag<G: EmissionGuarantee>(self, diag: &mut Diag<'_, G>) {
+        use MismatchedLifetimeSyntaxesSuggestion::*;
+
+        let style = |tool_only| {
+            if tool_only { SuggestionStyle::CompletelyHidden } else { SuggestionStyle::ShowAlways }
+        };
+
+        match self {
+            Implicit { suggestions, tool_only } => {
+                let suggestions = suggestions.into_iter().map(|s| (s, String::new())).collect();
+                diag.multipart_suggestion_with_style(
+                    fluent::lint_mismatched_lifetime_syntaxes_suggestion_implicit,
+                    suggestions,
+                    Applicability::MachineApplicable,
+                    style(tool_only),
+                );
+            }
+
+            Mixed { implicit_suggestions, explicit_anonymous_suggestions, tool_only } => {
+                let implicit_suggestions =
+                    implicit_suggestions.into_iter().map(|s| (s, String::new()));
+
+                let suggestions =
+                    implicit_suggestions.chain(explicit_anonymous_suggestions).collect();
+
+                diag.multipart_suggestion_with_style(
+                    fluent::lint_mismatched_lifetime_syntaxes_suggestion_mixed,
+                    suggestions,
+                    Applicability::MachineApplicable,
+                    style(tool_only),
+                );
+            }
+
+            Explicit { lifetime_name, suggestions, tool_only } => {
+                diag.arg("lifetime_name", lifetime_name);
+
+                let msg = diag.eagerly_translate(
+                    fluent::lint_mismatched_lifetime_syntaxes_suggestion_explicit,
+                );
+
+                diag.multipart_suggestion_with_style(
+                    msg,
+                    suggestions,
+                    Applicability::MachineApplicable,
+                    style(tool_only),
+                );
+            }
+        }
+    }
+}
diff --git a/compiler/rustc_lint/src/non_local_def.rs b/compiler/rustc_lint/src/non_local_def.rs
index 9ed11d9cc82..b877f909fc0 100644
--- a/compiler/rustc_lint/src/non_local_def.rs
+++ b/compiler/rustc_lint/src/non_local_def.rs
@@ -183,7 +183,7 @@ impl<'tcx> LateLintPass<'tcx> for NonLocalDefinitions {
                     && parent_opt_item_name != Some(kw::Underscore)
                     && let Some(parent) = parent.as_local()
                     && let Node::Item(item) = cx.tcx.hir_node_by_def_id(parent)
-                    && let ItemKind::Const(ident, ty, _, _) = item.kind
+                    && let ItemKind::Const(ident, _, ty, _) = item.kind
                     && let TyKind::Tup(&[]) = ty.kind
                 {
                     Some(ident.span)
diff --git a/compiler/rustc_lint/src/nonstandard_style.rs b/compiler/rustc_lint/src/nonstandard_style.rs
index 048d377b78f..31c18074466 100644
--- a/compiler/rustc_lint/src/nonstandard_style.rs
+++ b/compiler/rustc_lint/src/nonstandard_style.rs
@@ -513,7 +513,7 @@ impl<'tcx> LateLintPass<'tcx> for NonUpperCaseGlobals {
     fn check_item(&mut self, cx: &LateContext<'_>, it: &hir::Item<'_>) {
         let attrs = cx.tcx.hir_attrs(it.hir_id());
         match it.kind {
-            hir::ItemKind::Static(ident, ..)
+            hir::ItemKind::Static(_, ident, ..)
                 if !ast::attr::contains_name(attrs, sym::no_mangle) =>
             {
                 NonUpperCaseGlobals::check_upper_case(cx, "static variable", &ident);
diff --git a/compiler/rustc_lint/src/transmute.rs b/compiler/rustc_lint/src/transmute.rs
new file mode 100644
index 00000000000..bc1d4587d07
--- /dev/null
+++ b/compiler/rustc_lint/src/transmute.rs
@@ -0,0 +1,278 @@
+use rustc_errors::Applicability;
+use rustc_hir::def::{DefKind, Res};
+use rustc_hir::def_id::LocalDefId;
+use rustc_hir::{self as hir};
+use rustc_macros::LintDiagnostic;
+use rustc_middle::ty::{self, Ty};
+use rustc_session::{declare_lint, impl_lint_pass};
+use rustc_span::sym;
+
+use crate::{LateContext, LateLintPass};
+
+declare_lint! {
+    /// The `ptr_to_integer_transmute_in_consts` lint detects pointer to integer
+    /// transmute in const functions and associated constants.
+    ///
+    /// ### Example
+    ///
+    /// ```rust
+    /// const fn foo(ptr: *const u8) -> usize {
+    ///    unsafe {
+    ///        std::mem::transmute::<*const u8, usize>(ptr)
+    ///    }
+    /// }
+    /// ```
+    ///
+    /// {{produces}}
+    ///
+    /// ### Explanation
+    ///
+    /// Transmuting pointers to integers in a `const` context is undefined behavior.
+    /// Any attempt to use the resulting integer will abort const-evaluation.
+    ///
+    /// But sometimes the compiler might not emit an error for pointer to integer transmutes
+    /// inside const functions and associated consts because they are evaluated only when referenced.
+    /// Therefore, this lint serves as an extra layer of defense to prevent any undefined behavior
+    /// from compiling without any warnings or errors.
+    ///
+    /// See [std::mem::transmute] in the reference for more details.
+    ///
+    /// [std::mem::transmute]: https://doc.rust-lang.org/std/mem/fn.transmute.html
+    pub PTR_TO_INTEGER_TRANSMUTE_IN_CONSTS,
+    Warn,
+    "detects pointer to integer transmutes in const functions and associated constants",
+}
+
+declare_lint! {
+    /// The `unnecessary_transmutes` lint detects transmutations that have safer alternatives.
+    ///
+    /// ### Example
+    ///
+    /// ```rust
+    /// fn bytes_at_home(x: [u8; 4]) -> u32 {
+    ///   unsafe { std::mem::transmute(x) }
+    /// }
+    /// ```
+    ///
+    /// {{produces}}
+    ///
+    /// ### Explanation
+    ///
+    /// Using an explicit method is preferable over calls to
+    /// [`transmute`](https://doc.rust-lang.org/std/mem/fn.transmute.html) as
+    /// they more clearly communicate the intent, are easier to review, and
+    /// are less likely to accidentally result in unsoundness.
+    pub UNNECESSARY_TRANSMUTES,
+    Warn,
+    "detects transmutes that can also be achieved by other operations"
+}
+
+pub(crate) struct CheckTransmutes;
+
+impl_lint_pass!(CheckTransmutes => [PTR_TO_INTEGER_TRANSMUTE_IN_CONSTS, UNNECESSARY_TRANSMUTES]);
+
+impl<'tcx> LateLintPass<'tcx> for CheckTransmutes {
+    fn check_expr(&mut self, cx: &LateContext<'tcx>, expr: &'tcx hir::Expr<'tcx>) {
+        let hir::ExprKind::Call(callee, [arg]) = expr.kind else {
+            return;
+        };
+        let hir::ExprKind::Path(qpath) = callee.kind else {
+            return;
+        };
+        let Res::Def(DefKind::Fn, def_id) = cx.qpath_res(&qpath, callee.hir_id) else {
+            return;
+        };
+        if !cx.tcx.is_intrinsic(def_id, sym::transmute) {
+            return;
+        };
+        let body_owner_def_id = cx.tcx.hir_enclosing_body_owner(expr.hir_id);
+        let const_context = cx.tcx.hir_body_const_context(body_owner_def_id);
+        let args = cx.typeck_results().node_args(callee.hir_id);
+
+        let src = args.type_at(0);
+        let dst = args.type_at(1);
+
+        check_ptr_transmute_in_const(cx, expr, body_owner_def_id, const_context, src, dst);
+        check_unnecessary_transmute(cx, expr, callee, arg, const_context, src, dst);
+    }
+}
+
+/// Check for transmutes that exhibit undefined behavior.
+/// For example, transmuting pointers to integers in a const context.
+///
+/// Why do we consider const functions and associated constants only?
+///
+/// Generally, undefined behavior in const items are handled by the evaluator.
+/// But, const functions and associated constants are evaluated only when referenced.
+/// This can result in undefined behavior in a library going unnoticed until
+/// the function or constant is actually used.
+///
+/// Therefore, we only consider const functions and associated constants here and leave
+/// other const items to be handled by the evaluator.
+fn check_ptr_transmute_in_const<'tcx>(
+    cx: &LateContext<'tcx>,
+    expr: &'tcx hir::Expr<'tcx>,
+    body_owner_def_id: LocalDefId,
+    const_context: Option<hir::ConstContext>,
+    src: Ty<'tcx>,
+    dst: Ty<'tcx>,
+) {
+    if matches!(const_context, Some(hir::ConstContext::ConstFn))
+        || matches!(cx.tcx.def_kind(body_owner_def_id), DefKind::AssocConst)
+    {
+        if src.is_raw_ptr() && dst.is_integral() {
+            cx.tcx.emit_node_span_lint(
+                PTR_TO_INTEGER_TRANSMUTE_IN_CONSTS,
+                expr.hir_id,
+                expr.span,
+                UndefinedTransmuteLint,
+            );
+        }
+    }
+}
+
+/// Check for transmutes that overlap with stdlib methods.
+/// For example, transmuting `[u8; 4]` to `u32`.
+///
+/// We chose not to lint u8 -> bool transmutes, see #140431.
+fn check_unnecessary_transmute<'tcx>(
+    cx: &LateContext<'tcx>,
+    expr: &'tcx hir::Expr<'tcx>,
+    callee: &'tcx hir::Expr<'tcx>,
+    arg: &'tcx hir::Expr<'tcx>,
+    const_context: Option<hir::ConstContext>,
+    src: Ty<'tcx>,
+    dst: Ty<'tcx>,
+) {
+    let callee_span = callee.span.find_ancestor_inside(expr.span).unwrap_or(callee.span);
+    let (sugg, help) = match (src.kind(), dst.kind()) {
+        // dont check the length; transmute does that for us.
+        // [u8; _] => primitive
+        (ty::Array(t, _), ty::Uint(_) | ty::Float(_) | ty::Int(_))
+            if *t.kind() == ty::Uint(ty::UintTy::U8) =>
+        {
+            (
+                Some(vec![(callee_span, format!("{dst}::from_ne_bytes"))]),
+                Some(
+                    "there's also `from_le_bytes` and `from_be_bytes` if you expect a particular byte order",
+                ),
+            )
+        }
+        // primitive => [u8; _]
+        (ty::Uint(_) | ty::Float(_) | ty::Int(_), ty::Array(t, _))
+            if *t.kind() == ty::Uint(ty::UintTy::U8) =>
+        {
+            (
+                Some(vec![(callee_span, format!("{src}::to_ne_bytes"))]),
+                Some(
+                    "there's also `to_le_bytes` and `to_be_bytes` if you expect a particular byte order",
+                ),
+            )
+        }
+        // char → u32
+        (ty::Char, ty::Uint(ty::UintTy::U32)) => {
+            (Some(vec![(callee_span, "u32::from".to_string())]), None)
+        }
+        // char (→ u32) → i32
+        (ty::Char, ty::Int(ty::IntTy::I32)) => (
+            Some(vec![
+                (callee_span, "u32::from".to_string()),
+                (expr.span.shrink_to_hi(), ".cast_signed()".to_string()),
+            ]),
+            None,
+        ),
+        // u32 → char
+        (ty::Uint(ty::UintTy::U32), ty::Char) => (
+            Some(vec![(callee_span, "char::from_u32_unchecked".to_string())]),
+            Some("consider using `char::from_u32(…).unwrap()`"),
+        ),
+        // i32 → char
+        (ty::Int(ty::IntTy::I32), ty::Char) => (
+            Some(vec![
+                (callee_span, "char::from_u32_unchecked(i32::cast_unsigned".to_string()),
+                (expr.span.shrink_to_hi(), ")".to_string()),
+            ]),
+            Some("consider using `char::from_u32(i32::cast_unsigned(…)).unwrap()`"),
+        ),
+        // uNN → iNN
+        (ty::Uint(_), ty::Int(_)) => {
+            (Some(vec![(callee_span, format!("{src}::cast_signed"))]), None)
+        }
+        // iNN → uNN
+        (ty::Int(_), ty::Uint(_)) => {
+            (Some(vec![(callee_span, format!("{src}::cast_unsigned"))]), None)
+        }
+        // fNN → usize, isize
+        (ty::Float(_), ty::Uint(ty::UintTy::Usize) | ty::Int(ty::IntTy::Isize)) => (
+            Some(vec![
+                (callee_span, format!("{src}::to_bits")),
+                (expr.span.shrink_to_hi(), format!(" as {dst}")),
+            ]),
+            None,
+        ),
+        // fNN (→ uNN) → iNN
+        (ty::Float(_), ty::Int(..)) => (
+            Some(vec![
+                (callee_span, format!("{src}::to_bits")),
+                (expr.span.shrink_to_hi(), ".cast_signed()".to_string()),
+            ]),
+            None,
+        ),
+        // fNN → uNN
+        (ty::Float(_), ty::Uint(..)) => {
+            (Some(vec![(callee_span, format!("{src}::to_bits"))]), None)
+        }
+        // xsize → fNN
+        (ty::Uint(ty::UintTy::Usize) | ty::Int(ty::IntTy::Isize), ty::Float(_)) => (
+            Some(vec![
+                (callee_span, format!("{dst}::from_bits")),
+                (arg.span.shrink_to_hi(), " as _".to_string()),
+            ]),
+            None,
+        ),
+        // iNN (→ uNN) → fNN
+        (ty::Int(_), ty::Float(_)) => (
+            Some(vec![
+                (callee_span, format!("{dst}::from_bits({src}::cast_unsigned")),
+                (expr.span.shrink_to_hi(), ")".to_string()),
+            ]),
+            None,
+        ),
+        // uNN → fNN
+        (ty::Uint(_), ty::Float(_)) => {
+            (Some(vec![(callee_span, format!("{dst}::from_bits"))]), None)
+        }
+        // bool → x8 in const context since `From::from` is not const yet
+        // FIXME: Consider arg expr's precedence to avoid parentheses.
+        // FIXME(const_traits): Remove this when `From::from` is constified.
+        (ty::Bool, ty::Int(..) | ty::Uint(..)) if const_context.is_some() => (
+            Some(vec![
+                (callee_span, "".to_string()),
+                (expr.span.shrink_to_hi(), format!(" as {dst}")),
+            ]),
+            None,
+        ),
+        // bool → x8 using `x8::from`
+        (ty::Bool, ty::Int(..) | ty::Uint(..)) => {
+            (Some(vec![(callee_span, format!("{dst}::from"))]), None)
+        }
+        _ => return,
+    };
+
+    cx.tcx.node_span_lint(UNNECESSARY_TRANSMUTES, expr.hir_id, expr.span, |diag| {
+        diag.primary_message("unnecessary transmute");
+        if let Some(sugg) = sugg {
+            diag.multipart_suggestion("replace this with", sugg, Applicability::MachineApplicable);
+        }
+        if let Some(help) = help {
+            diag.help(help);
+        }
+    });
+}
+
+#[derive(LintDiagnostic)]
+#[diag(lint_undefined_transmute)]
+#[note]
+#[note(lint_note2)]
+#[help]
+pub(crate) struct UndefinedTransmuteLint;
diff --git a/compiler/rustc_lint/src/types.rs b/compiler/rustc_lint/src/types.rs
index f1c06dfe6ce..fec23354c91 100644
--- a/compiler/rustc_lint/src/types.rs
+++ b/compiler/rustc_lint/src/types.rs
@@ -1,9 +1,7 @@
 use std::iter;
 use std::ops::ControlFlow;
 
-use rustc_abi::{
-    BackendRepr, Integer, IntegerType, TagEncoding, VariantIdx, Variants, WrappingRange,
-};
+use rustc_abi::{BackendRepr, TagEncoding, VariantIdx, Variants, WrappingRange};
 use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::DiagMessage;
 use rustc_hir::intravisit::VisitorExt;
@@ -24,7 +22,8 @@ mod improper_ctypes;
 
 use crate::lints::{
     AmbiguousWidePointerComparisons, AmbiguousWidePointerComparisonsAddrMetadataSuggestion,
-    AmbiguousWidePointerComparisonsAddrSuggestion, AtomicOrderingFence, AtomicOrderingLoad,
+    AmbiguousWidePointerComparisonsAddrSuggestion, AmbiguousWidePointerComparisonsCastSuggestion,
+    AmbiguousWidePointerComparisonsExpectSuggestion, AtomicOrderingFence, AtomicOrderingLoad,
     AtomicOrderingStore, ImproperCTypes, InvalidAtomicOrderingDiag, InvalidNanComparisons,
     InvalidNanComparisonsSuggestion, UnpredictableFunctionPointerComparisons,
     UnpredictableFunctionPointerComparisonsSuggestion, UnusedComparisons, UsesPowerAlignment,
@@ -362,6 +361,7 @@ fn lint_wide_pointer<'tcx>(
     let ne = if cmpop == ComparisonOp::BinOp(hir::BinOpKind::Ne) { "!" } else { "" };
     let is_eq_ne = matches!(cmpop, ComparisonOp::BinOp(hir::BinOpKind::Eq | hir::BinOpKind::Ne));
     let is_dyn_comparison = l_inner_ty_is_dyn && r_inner_ty_is_dyn;
+    let via_method_call = matches!(&e.kind, ExprKind::MethodCall(..) | ExprKind::Call(..));
 
     let left = e.span.shrink_to_lo().until(l_span.shrink_to_lo());
     let middle = l_span.shrink_to_hi().until(r_span.shrink_to_lo());
@@ -376,21 +376,21 @@ fn lint_wide_pointer<'tcx>(
     cx.emit_span_lint(
         AMBIGUOUS_WIDE_POINTER_COMPARISONS,
         e.span,
-        AmbiguousWidePointerComparisons::Spanful {
-            addr_metadata_suggestion: (is_eq_ne && !is_dyn_comparison).then(|| {
-                AmbiguousWidePointerComparisonsAddrMetadataSuggestion {
-                    ne,
-                    deref_left,
-                    deref_right,
-                    l_modifiers,
-                    r_modifiers,
-                    left,
-                    middle,
-                    right,
-                }
-            }),
-            addr_suggestion: if is_eq_ne {
-                AmbiguousWidePointerComparisonsAddrSuggestion::AddrEq {
+        if is_eq_ne {
+            AmbiguousWidePointerComparisons::SpanfulEq {
+                addr_metadata_suggestion: (!is_dyn_comparison).then(|| {
+                    AmbiguousWidePointerComparisonsAddrMetadataSuggestion {
+                        ne,
+                        deref_left,
+                        deref_right,
+                        l_modifiers,
+                        r_modifiers,
+                        left,
+                        middle,
+                        right,
+                    }
+                }),
+                addr_suggestion: AmbiguousWidePointerComparisonsAddrSuggestion {
                     ne,
                     deref_left,
                     deref_right,
@@ -399,9 +399,11 @@ fn lint_wide_pointer<'tcx>(
                     left,
                     middle,
                     right,
-                }
-            } else {
-                AmbiguousWidePointerComparisonsAddrSuggestion::Cast {
+                },
+            }
+        } else {
+            AmbiguousWidePointerComparisons::SpanfulCmp {
+                cast_suggestion: AmbiguousWidePointerComparisonsCastSuggestion {
                     deref_left,
                     deref_right,
                     l_modifiers,
@@ -412,8 +414,14 @@ fn lint_wide_pointer<'tcx>(
                     left_after: l_span.shrink_to_hi(),
                     right_before: (r_ty_refs != 0).then_some(r_span.shrink_to_lo()),
                     right_after: r_span.shrink_to_hi(),
-                }
-            },
+                },
+                expect_suggestion: AmbiguousWidePointerComparisonsExpectSuggestion {
+                    paren_left: if via_method_call { "" } else { "(" },
+                    paren_right: if via_method_call { "" } else { ")" },
+                    before: e.span.shrink_to_lo(),
+                    after: e.span.shrink_to_hi(),
+                },
+            }
         },
     );
 }
@@ -1274,14 +1282,6 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
                             };
                         }
 
-                        if let Some(IntegerType::Fixed(Integer::I128, _)) = def.repr().int {
-                            return FfiUnsafe {
-                                ty,
-                                reason: fluent::lint_improper_ctypes_128bit,
-                                help: None,
-                            };
-                        }
-
                         use improper_ctypes::check_non_exhaustive_variant;
 
                         let non_exhaustive = def.variant_list_has_applicable_non_exhaustive();
@@ -1314,10 +1314,6 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
             // but only the base type is relevant for being representable in FFI.
             ty::Pat(base, ..) => self.check_type_for_ffi(acc, base),
 
-            ty::Int(ty::IntTy::I128) | ty::Uint(ty::UintTy::U128) => {
-                FfiUnsafe { ty, reason: fluent::lint_improper_ctypes_128bit, help: None }
-            }
-
             // Primitive types with a stable representation.
             ty::Bool | ty::Int(..) | ty::Uint(..) | ty::Float(..) | ty::Never => FfiSafe,
 
@@ -1700,7 +1696,7 @@ impl ImproperCTypesDefinitions {
             && cx.tcx.sess.target.os == "aix"
             && !adt_def.all_fields().next().is_none()
         {
-            let struct_variant_data = item.expect_struct().1;
+            let struct_variant_data = item.expect_struct().2;
             for field_def in struct_variant_data.fields().iter().skip(1) {
                 // Struct fields (after the first field) are checked for the
                 // power alignment rule, as fields after the first are likely
@@ -1725,9 +1721,9 @@ impl ImproperCTypesDefinitions {
 impl<'tcx> LateLintPass<'tcx> for ImproperCTypesDefinitions {
     fn check_item(&mut self, cx: &LateContext<'tcx>, item: &'tcx hir::Item<'tcx>) {
         match item.kind {
-            hir::ItemKind::Static(_, ty, ..)
-            | hir::ItemKind::Const(_, ty, ..)
-            | hir::ItemKind::TyAlias(_, ty, ..) => {
+            hir::ItemKind::Static(_, _, ty, _)
+            | hir::ItemKind::Const(_, _, ty, _)
+            | hir::ItemKind::TyAlias(_, _, ty) => {
                 self.check_ty_maybe_containing_foreign_fnptr(
                     cx,
                     ty,
@@ -1794,7 +1790,7 @@ declare_lint_pass!(VariantSizeDifferences => [VARIANT_SIZE_DIFFERENCES]);
 
 impl<'tcx> LateLintPass<'tcx> for VariantSizeDifferences {
     fn check_item(&mut self, cx: &LateContext<'_>, it: &hir::Item<'_>) {
-        if let hir::ItemKind::Enum(_, ref enum_definition, _) = it.kind {
+        if let hir::ItemKind::Enum(_, _, ref enum_definition) = it.kind {
             let t = cx.tcx.type_of(it.owner_id).instantiate_identity();
             let ty = cx.tcx.erase_regions(t);
             let Ok(layout) = cx.layout_of(ty) else { return };
diff --git a/compiler/rustc_lint/src/unqualified_local_imports.rs b/compiler/rustc_lint/src/unqualified_local_imports.rs
index 50c5119285f..0076cae3cff 100644
--- a/compiler/rustc_lint/src/unqualified_local_imports.rs
+++ b/compiler/rustc_lint/src/unqualified_local_imports.rs
@@ -1,4 +1,3 @@
-use rustc_hir::def::{DefKind, Res};
 use rustc_hir::{self as hir};
 use rustc_session::{declare_lint, declare_lint_pass};
 use rustc_span::kw;
@@ -47,17 +46,15 @@ declare_lint_pass!(UnqualifiedLocalImports => [UNQUALIFIED_LOCAL_IMPORTS]);
 impl<'tcx> LateLintPass<'tcx> for UnqualifiedLocalImports {
     fn check_item(&mut self, cx: &LateContext<'tcx>, item: &'tcx hir::Item<'tcx>) {
         let hir::ItemKind::Use(path, _kind) = item.kind else { return };
-        // `path` has three resolutions for the type, module, value namespaces.
-        // Check if any of them qualifies: local crate, and not a macro.
-        // (Macros can't be imported any other way so we don't complain about them.)
-        let is_local_import = |res: &Res| {
-            matches!(
-                res,
-                hir::def::Res::Def(def_kind, def_id)
-                    if def_id.is_local() && !matches!(def_kind, DefKind::Macro(_)),
-            )
-        };
-        if !path.res.iter().any(is_local_import) {
+        // Check the type and value namespace resolutions for a local crate.
+        let is_local_import = matches!(
+            path.res.type_ns,
+            Some(hir::def::Res::Def(_, def_id)) if def_id.is_local()
+        ) || matches!(
+            path.res.value_ns,
+            Some(hir::def::Res::Def(_, def_id)) if def_id.is_local()
+        );
+        if !is_local_import {
             return;
         }
         // So this does refer to something local. Let's check whether it starts with `self`,
diff --git a/compiler/rustc_lint/src/unused.rs b/compiler/rustc_lint/src/unused.rs
index 50a27d7e84f..1620f425794 100644
--- a/compiler/rustc_lint/src/unused.rs
+++ b/compiler/rustc_lint/src/unused.rs
@@ -1,8 +1,7 @@
 use std::iter;
 
-use rustc_ast as ast;
 use rustc_ast::util::{classify, parser};
-use rustc_ast::{ExprKind, StmtKind};
+use rustc_ast::{self as ast, ExprKind, HasAttrs as _, StmtKind};
 use rustc_errors::{MultiSpan, pluralize};
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::DefId;
@@ -780,26 +779,30 @@ trait UnusedDelimLint {
         right_pos: Option<BytePos>,
         is_kw: bool,
     ) {
-        let spans = match value.kind {
-            ast::ExprKind::Block(ref block, None) if let [stmt] = block.stmts.as_slice() => stmt
-                .span
-                .find_ancestor_inside(value.span)
-                .map(|span| (value.span.with_hi(span.lo()), value.span.with_lo(span.hi()))),
+        let span_with_attrs = match value.kind {
+            ast::ExprKind::Block(ref block, None) if let [stmt] = block.stmts.as_slice() => {
+                // For the statements with attributes, like `{ #[allow()] println!("Hello!") }`,
+                // the span should contains the attributes, or the suggestion will remove them.
+                if let Some(attr_lo) = stmt.attrs().iter().map(|attr| attr.span.lo()).min() {
+                    stmt.span.with_lo(attr_lo)
+                } else {
+                    stmt.span
+                }
+            }
             ast::ExprKind::Paren(ref expr) => {
                 // For the expr with attributes, like `let _ = (#[inline] || println!("Hello!"));`,
                 // the span should contains the attributes, or the suggestion will remove them.
-                let expr_span_with_attrs =
-                    if let Some(attr_lo) = expr.attrs.iter().map(|attr| attr.span.lo()).min() {
-                        expr.span.with_lo(attr_lo)
-                    } else {
-                        expr.span
-                    };
-                expr_span_with_attrs.find_ancestor_inside(value.span).map(|expr_span| {
-                    (value.span.with_hi(expr_span.lo()), value.span.with_lo(expr_span.hi()))
-                })
+                if let Some(attr_lo) = expr.attrs.iter().map(|attr| attr.span.lo()).min() {
+                    expr.span.with_lo(attr_lo)
+                } else {
+                    expr.span
+                }
             }
             _ => return,
         };
+        let spans = span_with_attrs
+            .find_ancestor_inside(value.span)
+            .map(|span| (value.span.with_hi(span.lo()), value.span.with_lo(span.hi())));
         let keep_space = (
             left_pos.is_some_and(|s| s >= value.span.lo()),
             right_pos.is_some_and(|s| s <= value.span.hi()),
diff --git a/compiler/rustc_lint_defs/src/builtin.rs b/compiler/rustc_lint_defs/src/builtin.rs
index 3cea24634fe..777118e69fb 100644
--- a/compiler/rustc_lint_defs/src/builtin.rs
+++ b/compiler/rustc_lint_defs/src/builtin.rs
@@ -16,6 +16,7 @@ declare_lint_pass! {
     /// that are used by other parts of the compiler.
     HardwiredLints => [
         // tidy-alphabetical-start
+        AARCH64_SOFTFLOAT_NEON,
         ABSOLUTE_PATHS_NOT_STARTING_WITH_CRATE,
         AMBIGUOUS_ASSOCIATED_ITEMS,
         AMBIGUOUS_GLOB_IMPORTS,
@@ -39,7 +40,6 @@ declare_lint_pass! {
         DUPLICATE_MACRO_ATTRIBUTES,
         ELIDED_LIFETIMES_IN_ASSOCIATED_CONSTANT,
         ELIDED_LIFETIMES_IN_PATHS,
-        ELIDED_NAMED_LIFETIMES,
         EXPLICIT_BUILTIN_CFGS_IN_FLAGS,
         EXPORTED_PRIVATE_DEPENDENCIES,
         FFI_UNWIND_CALLS,
@@ -78,7 +78,6 @@ declare_lint_pass! {
         PRIVATE_BOUNDS,
         PRIVATE_INTERFACES,
         PROC_MACRO_DERIVE_RESOLUTION_FALLBACK,
-        PTR_TO_INTEGER_TRANSMUTE_IN_CONSTS,
         PUB_USE_OF_PRIVATE_EXTERN_CRATE,
         REDUNDANT_IMPORTS,
         REDUNDANT_LIFETIMES,
@@ -103,6 +102,7 @@ declare_lint_pass! {
         TAIL_EXPR_DROP_ORDER,
         TEST_UNSTABLE_LINT,
         TEXT_DIRECTION_CODEPOINT_IN_COMMENT,
+        TEXT_DIRECTION_CODEPOINT_IN_LITERAL,
         TRIVIAL_CASTS,
         TRIVIAL_NUMERIC_CASTS,
         TYVAR_BEHIND_RAW_POINTER,
@@ -117,7 +117,6 @@ declare_lint_pass! {
         UNKNOWN_OR_MALFORMED_DIAGNOSTIC_ATTRIBUTES,
         UNNAMEABLE_TEST_ITEMS,
         UNNAMEABLE_TYPES,
-        UNNECESSARY_TRANSMUTES,
         UNREACHABLE_CODE,
         UNREACHABLE_PATTERNS,
         UNSAFE_ATTR_OUTSIDE_UNSAFE,
@@ -178,8 +177,9 @@ declare_lint! {
     Warn,
     "applying forbid to lint-groups",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #81670 <https://github.com/rust-lang/rust/issues/81670>",
+        report_in_deps: true,
     };
 }
 
@@ -214,7 +214,7 @@ declare_lint! {
     Deny,
     "ill-formed attribute inputs that were previously accepted and used in practice",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #57571 <https://github.com/rust-lang/rust/issues/57571>",
     };
     crate_level_only
@@ -251,8 +251,9 @@ declare_lint! {
     Deny,
     "conflicts between `#[repr(..)]` hints that were previously accepted and used in practice",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #68585 <https://github.com/rust-lang/rust/issues/68585>",
+        report_in_deps: true,
     };
 }
 
@@ -1240,8 +1241,9 @@ declare_lint! {
     Deny,
     "detect public re-exports of private extern crates",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #127909 <https://github.com/rust-lang/rust/issues/127909>",
+        report_in_deps: true,
     };
 }
 
@@ -1270,8 +1272,9 @@ declare_lint! {
     Deny,
     "type parameter default erroneously allowed in invalid location",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #36887 <https://github.com/rust-lang/rust/issues/36887>",
+        report_in_deps: true,
     };
 }
 
@@ -1409,7 +1412,7 @@ declare_lint! {
     Deny,
     "patterns in functions without body were erroneously allowed",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #35203 <https://github.com/rust-lang/rust/issues/35203>",
     };
 }
@@ -1453,8 +1456,9 @@ declare_lint! {
     Deny,
     "detects missing fragment specifiers in unused `macro_rules!` patterns",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #40107 <https://github.com/rust-lang/rust/issues/40107>",
+        report_in_deps: true,
     };
 }
 
@@ -1495,7 +1499,7 @@ declare_lint! {
     Warn,
     "detects generic lifetime arguments in path segments with late bound lifetime parameters",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #42868 <https://github.com/rust-lang/rust/issues/42868>",
     };
 }
@@ -1828,38 +1832,6 @@ declare_lint! {
 }
 
 declare_lint! {
-    /// The `elided_named_lifetimes` lint detects when an elided
-    /// lifetime ends up being a named lifetime, such as `'static`
-    /// or some lifetime parameter `'a`.
-    ///
-    /// ### Example
-    ///
-    /// ```rust,compile_fail
-    /// #![deny(elided_named_lifetimes)]
-    /// struct Foo;
-    /// impl Foo {
-    ///     pub fn get_mut(&'static self, x: &mut u8) -> &mut u8 {
-    ///         unsafe { &mut *(x as *mut _) }
-    ///     }
-    /// }
-    /// ```
-    ///
-    /// {{produces}}
-    ///
-    /// ### Explanation
-    ///
-    /// Lifetime elision is quite useful, because it frees you from having
-    /// to give each lifetime its own name, but sometimes it can produce
-    /// somewhat surprising resolutions. In safe code, it is mostly okay,
-    /// because the borrow checker prevents any unsoundness, so the worst
-    /// case scenario is you get a confusing error message in some other place.
-    /// But with `unsafe` code, such unexpected resolutions may lead to unsound code.
-    pub ELIDED_NAMED_LIFETIMES,
-    Warn,
-    "detects when an elided lifetime gets resolved to be `'static` or some named parameter"
-}
-
-declare_lint! {
     /// The `bare_trait_objects` lint suggests using `dyn Trait` for trait
     /// objects.
     ///
@@ -2122,8 +2094,9 @@ declare_lint! {
     Deny,
     "detects proc macro derives using inaccessible names from parent modules",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #83583 <https://github.com/rust-lang/rust/issues/83583>",
+        report_in_deps: true,
     };
 }
 
@@ -2225,7 +2198,7 @@ declare_lint! {
     "macro-expanded `macro_export` macros from the current crate \
      cannot be referred to by absolute paths",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #52234 <https://github.com/rust-lang/rust/issues/52234>",
     };
     crate_level_only
@@ -2346,7 +2319,7 @@ declare_lint! {
     Deny,
     "ambiguous associated items",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #57644 <https://github.com/rust-lang/rust/issues/57644>",
     };
 }
@@ -2362,8 +2335,9 @@ declare_lint! {
     Deny,
     "a feature gate that doesn't break dependent crates",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #64266 <https://github.com/rust-lang/rust/issues/64266>",
+        report_in_deps: true,
     };
 }
 
@@ -2674,7 +2648,7 @@ declare_lint! {
     Warn,
     "detects a generic constant is used in a type without a emitting a warning",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #76200 <https://github.com/rust-lang/rust/issues/76200>",
     };
 }
@@ -2733,7 +2707,7 @@ declare_lint! {
     Warn,
     "uninhabited static",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #74840 <https://github.com/rust-lang/rust/issues/74840>",
     };
 }
@@ -2866,7 +2840,7 @@ declare_lint! {
     Warn,
     "detect unsupported use of `Self` from outer item",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #124186 <https://github.com/rust-lang/rust/issues/124186>",
     };
 }
@@ -2912,8 +2886,9 @@ declare_lint! {
     Warn,
     "trailing semicolon in macro body used as expression",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #79813 <https://github.com/rust-lang/rust/issues/79813>",
+        report_in_deps: true,
     };
 }
 
@@ -2959,7 +2934,7 @@ declare_lint! {
     Warn,
     "detects derive helper attributes that are used before they are introduced",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #79202 <https://github.com/rust-lang/rust/issues/79202>",
     };
 }
@@ -3126,7 +3101,7 @@ declare_lint! {
     Warn,
     "transparent type contains an external ZST that is marked #[non_exhaustive] or contains private fields",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #78586 <https://github.com/rust-lang/rust/issues/78586>",
     };
 }
@@ -3177,7 +3152,7 @@ declare_lint! {
     Warn,
     "unstable syntax can change at any point in the future, causing a hard error!",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #65860 <https://github.com/rust-lang/rust/issues/65860>",
     };
 }
@@ -3647,6 +3622,54 @@ declare_lint! {
 }
 
 declare_lint! {
+    /// The `unsupported_calling_conventions` lint is output whenever there is a use of the
+    /// `stdcall`, `fastcall`, and `cdecl` calling conventions (or their unwind
+    /// variants) on targets that cannot meaningfully be supported for the requested target.
+    ///
+    /// For example `stdcall` does not make much sense for a x86_64 or, more apparently, powerpc
+    /// code, because this calling convention was never specified for those targets.
+    ///
+    /// Historically MSVC toolchains have fallen back to the regular C calling convention for
+    /// targets other than x86, but Rust doesn't really see a similar need to introduce a similar
+    /// hack across many more targets.
+    ///
+    /// ### Example
+    ///
+    /// ```rust,ignore (needs specific targets)
+    /// extern "stdcall" fn stdcall() {}
+    /// ```
+    ///
+    /// This will produce:
+    ///
+    /// ```text
+    /// warning: use of calling convention not supported on this target
+    ///   --> $DIR/unsupported.rs:39:1
+    ///    |
+    /// LL | extern "stdcall" fn stdcall() {}
+    ///    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+    ///    |
+    ///    = note: `#[warn(unsupported_calling_conventions)]` on by default
+    ///    = warning: this was previously accepted by the compiler but is being phased out;
+    ///               it will become a hard error in a future release!
+    ///    = note: for more information, see issue ...
+    /// ```
+    ///
+    /// ### Explanation
+    ///
+    /// On most of the targets the behaviour of `stdcall` and similar calling conventions is not
+    /// defined at all, but was previously accepted due to a bug in the implementation of the
+    /// compiler.
+    pub UNSUPPORTED_CALLING_CONVENTIONS,
+    Warn,
+    "use of unsupported calling convention",
+    @future_incompatible = FutureIncompatibleInfo {
+        reason: FutureIncompatibilityReason::FutureReleaseError,
+        report_in_deps: true,
+        reference: "issue #137018 <https://github.com/rust-lang/rust/issues/137018>",
+    };
+}
+
+declare_lint! {
     /// The `unsupported_fn_ptr_calling_conventions` lint is output whenever there is a use of
     /// a target dependent calling convention on a target that does not support this calling
     /// convention on a function pointer.
@@ -3685,8 +3708,9 @@ declare_lint! {
     Warn,
     "use of unsupported calling convention for function pointer",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #130260 <https://github.com/rust-lang/rust/issues/130260>",
+        report_in_deps: true,
     };
 }
 
@@ -3783,7 +3807,6 @@ declare_lint! {
 }
 
 declare_lint! {
-    #[allow(text_direction_codepoint_in_literal)]
     /// The `text_direction_codepoint_in_comment` lint detects Unicode codepoints in comments that
     /// change the visual representation of text on screen in a way that does not correspond to
     /// their on memory representation.
@@ -3793,7 +3816,7 @@ declare_lint! {
     /// ```rust,compile_fail
     /// #![deny(text_direction_codepoint_in_comment)]
     /// fn main() {
-    ///     println!("{:?}"); // '‮');
+    #[doc = "    println!(\"{:?}\"); // '\u{202E}');"]
     /// }
     /// ```
     ///
@@ -3808,7 +3831,43 @@ declare_lint! {
     /// their use.
     pub TEXT_DIRECTION_CODEPOINT_IN_COMMENT,
     Deny,
-    "invisible directionality-changing codepoints in comment"
+    "invisible directionality-changing codepoints in comment",
+    crate_level_only
+}
+
+declare_lint! {
+    /// The `text_direction_codepoint_in_literal` lint detects Unicode codepoints that change the
+    /// visual representation of text on screen in a way that does not correspond to their on
+    /// memory representation.
+    ///
+    /// ### Explanation
+    ///
+    /// The unicode characters `\u{202A}`, `\u{202B}`, `\u{202D}`, `\u{202E}`, `\u{2066}`,
+    /// `\u{2067}`, `\u{2068}`, `\u{202C}` and `\u{2069}` make the flow of text on screen change
+    /// its direction on software that supports these codepoints. This makes the text "abc" display
+    /// as "cba" on screen. By leveraging software that supports these, people can write specially
+    /// crafted literals that make the surrounding code seem like it's performing one action, when
+    /// in reality it is performing another. Because of this, we proactively lint against their
+    /// presence to avoid surprises.
+    ///
+    /// ### Example
+    ///
+    /// ```rust,compile_fail
+    /// #![deny(text_direction_codepoint_in_literal)]
+    /// fn main() {
+    // ` - convince tidy that backticks match
+    #[doc = "    println!(\"{:?}\", '\u{202E}');"]
+    // `
+    /// }
+    /// ```
+    ///
+    /// {{produces}}
+    ///
+    pub TEXT_DIRECTION_CODEPOINT_IN_LITERAL,
+    Deny,
+    "detect special Unicode codepoints that affect the visual representation of text on screen, \
+     changing the direction in which text flows",
+    crate_level_only
 }
 
 declare_lint! {
@@ -4333,7 +4392,7 @@ declare_lint! {
     Warn,
     "detects certain glob imports that require reporting an ambiguity error",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #114095 <https://github.com/rust-lang/rust/issues/114095>",
     };
 }
@@ -4488,7 +4547,7 @@ declare_lint! {
     Deny,
     "elided lifetimes cannot be used in associated constants in impls",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #115010 <https://github.com/rust-lang/rust/issues/115010>",
     };
 }
@@ -4535,7 +4594,7 @@ declare_lint! {
     Warn,
     "detects certain macro bindings that should not be re-exported",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #120192 <https://github.com/rust-lang/rust/issues/120192>",
     };
 }
@@ -4600,7 +4659,7 @@ declare_lint! {
     Warn,
     "impl contains type parameters that are not covered",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #124559 <https://github.com/rust-lang/rust/issues/124559>",
     };
 }
@@ -4764,7 +4823,7 @@ declare_lint! {
     Warn,
     "detects out of scope calls to `macro_rules` in key-value attributes",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #124535 <https://github.com/rust-lang/rust/issues/124535>",
     };
 }
@@ -4851,64 +4910,6 @@ declare_lint! {
 }
 
 declare_lint! {
-    /// The `ptr_to_integer_transmute_in_consts` lint detects pointer to integer
-    /// transmute in const functions and associated constants.
-    ///
-    /// ### Example
-    ///
-    /// ```rust
-    /// const fn foo(ptr: *const u8) -> usize {
-    ///    unsafe {
-    ///        std::mem::transmute::<*const u8, usize>(ptr)
-    ///    }
-    /// }
-    /// ```
-    ///
-    /// {{produces}}
-    ///
-    /// ### Explanation
-    ///
-    /// Transmuting pointers to integers in a `const` context is undefined behavior.
-    /// Any attempt to use the resulting integer will abort const-evaluation.
-    ///
-    /// But sometimes the compiler might not emit an error for pointer to integer transmutes
-    /// inside const functions and associated consts because they are evaluated only when referenced.
-    /// Therefore, this lint serves as an extra layer of defense to prevent any undefined behavior
-    /// from compiling without any warnings or errors.
-    ///
-    /// See [std::mem::transmute] in the reference for more details.
-    ///
-    /// [std::mem::transmute]: https://doc.rust-lang.org/std/mem/fn.transmute.html
-    pub PTR_TO_INTEGER_TRANSMUTE_IN_CONSTS,
-    Warn,
-    "detects pointer to integer transmutes in const functions and associated constants",
-}
-
-declare_lint! {
-    /// The `unnecessary_transmutes` lint detects transmutations that have safer alternatives.
-    ///
-    /// ### Example
-    ///
-    /// ```rust
-    /// fn bytes_at_home(x: [u8; 4]) -> u32 {
-    ///   unsafe { std::mem::transmute(x) }
-    /// }
-    /// ```
-    ///
-    /// {{produces}}
-    ///
-    /// ### Explanation
-    ///
-    /// Using an explicit method is preferable over calls to
-    /// [`transmute`](https://doc.rust-lang.org/std/mem/fn.transmute.html) as
-    /// they more clearly communicate the intent, are easier to review, and
-    /// are less likely to accidentally result in unsoundness.
-    pub UNNECESSARY_TRANSMUTES,
-    Warn,
-    "detects transmutes that are shadowed by std methods"
-}
-
-declare_lint! {
     /// The `tail_expr_drop_order` lint looks for those values generated at the tail expression location,
     /// that runs a custom `Drop` destructor.
     /// Some of them may be dropped earlier in Edition 2024 that they used to in Edition 2021 and prior.
@@ -5043,14 +5044,14 @@ declare_lint! {
     ///
     /// ```text
     /// error: this function function definition is affected by the wasm ABI transition: it passes an argument of non-scalar type `MyType`
-    /// --> $DIR/wasm_c_abi_transition.rs:17:1
-    ///  |
-    ///  | pub extern "C" fn my_fun(_x: MyType) {}
-    ///  | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-    ///  |
-    ///  = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!
-    ///  = note: for more information, see issue #138762 <https://github.com/rust-lang/rust/issues/138762>
-    ///  = help: the "C" ABI Rust uses on wasm32-unknown-unknown will change to align with the standard "C" ABI for this target
+    ///   --> $DIR/wasm_c_abi_transition.rs:17:1
+    ///    |
+    ///    | pub extern "C" fn my_fun(_x: MyType) {}
+    ///    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+    ///    |
+    ///    = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!
+    ///    = note: for more information, see issue #138762 <https://github.com/rust-lang/rust/issues/138762>
+    ///    = help: the "C" ABI Rust uses on wasm32-unknown-unknown will change to align with the standard "C" ABI for this target
     /// ```
     ///
     /// ### Explanation
@@ -5063,7 +5064,50 @@ declare_lint! {
     Warn,
     "detects code relying on rustc's non-spec-compliant wasm C ABI",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseError,
         reference: "issue #138762 <https://github.com/rust-lang/rust/issues/138762>",
+        report_in_deps: true,
+    };
+}
+
+declare_lint! {
+    /// The `aarch64_softfloat_neon` lint detects usage of `#[target_feature(enable = "neon")]` on
+    /// softfloat aarch64 targets. Enabling this target feature causes LLVM to alter the ABI of
+    /// function calls, making this attribute unsound to use.
+    ///
+    /// ### Example
+    ///
+    /// ```rust,ignore (needs aarch64-unknown-none-softfloat)
+    /// #[target_feature(enable = "neon")]
+    /// fn with_neon() {}
+    /// ```
+    ///
+    /// This will produce:
+    ///
+    /// ```text
+    /// error: enabling the `neon` target feature on the current target is unsound due to ABI issues
+    ///   --> $DIR/abi-incompatible-target-feature-attribute-fcw.rs:11:18
+    ///    |
+    ///    | #[target_feature(enable = "neon")]
+    ///    |                  ^^^^^^^^^^^^^^^
+    ///    |
+    ///    = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release!
+    ///    = note: for more information, see issue #134375 <https://github.com/rust-lang/rust/issues/134375>
+    /// ```
+    ///
+    /// ### Explanation
+    ///
+    /// If a function like `with_neon` above ends up containing calls to LLVM builtins, those will
+    /// not use the correct ABI. This is caused by a lack of support in LLVM for mixing code with
+    /// and without the `neon` target feature. The target feature should never have been stabilized
+    /// on this target due to this issue, but the problem was not known at the time of
+    /// stabilization.
+    pub AARCH64_SOFTFLOAT_NEON,
+    Warn,
+    "detects code that could be affected by ABI issues on aarch64 softfloat targets",
+    @future_incompatible = FutureIncompatibleInfo {
+        reason: FutureIncompatibilityReason::FutureReleaseError,
+        reference: "issue #134375 <https://github.com/rust-lang/rust/issues/134375>",
+        report_in_deps: true,
     };
 }
diff --git a/compiler/rustc_lint_defs/src/lib.rs b/compiler/rustc_lint_defs/src/lib.rs
index b4069b317bf..1d9b7a7fcb9 100644
--- a/compiler/rustc_lint_defs/src/lib.rs
+++ b/compiler/rustc_lint_defs/src/lib.rs
@@ -9,7 +9,7 @@ use rustc_data_structures::stable_hasher::{
 use rustc_error_messages::{DiagMessage, MultiSpan};
 use rustc_hir::def::Namespace;
 use rustc_hir::def_id::DefPathHash;
-use rustc_hir::{HashStableContext, HirId, ItemLocalId, MissingLifetimeKind};
+use rustc_hir::{HashStableContext, HirId, ItemLocalId};
 use rustc_macros::{Decodable, Encodable, HashStable_Generic};
 pub use rustc_span::edition::Edition;
 use rustc_span::{Ident, MacroRulesNormalizedIdent, Span, Symbol, sym};
@@ -361,6 +361,18 @@ pub struct FutureIncompatibleInfo {
     /// Set to false for lints that already include a more detailed
     /// explanation.
     pub explain_reason: bool,
+    /// If set to `true`, this will make future incompatibility warnings show up in cargo's
+    /// reports.
+    ///
+    /// When a future incompatibility warning is first inroduced, set this to `false`
+    /// (or, rather, don't override the default). This allows crate developers an opportunity
+    /// to fix the warning before blasting all dependents with a warning they can't fix
+    /// (dependents have to wait for a new release of the affected crate to be published).
+    ///
+    /// After a lint has been in this state for a while, consider setting this to true, so it
+    /// warns for everyone. It is a good signal that it is ready if you can determine that all
+    /// or most affected crates on crates.io have been updated.
+    pub report_in_deps: bool,
 }
 
 /// The reason for future incompatibility
@@ -380,46 +392,24 @@ pub struct FutureIncompatibleInfo {
 pub enum FutureIncompatibilityReason {
     /// This will be an error in a future release for all editions
     ///
-    /// This will *not* show up in cargo's future breakage report.
-    /// The warning will hence only be seen in local crates, not in dependencies.
-    ///
     /// Choose this variant when you are first introducing a "future
     /// incompatible" warning that is intended to eventually be fixed in the
-    /// future. This allows crate developers an opportunity to fix the warning
-    /// before blasting all dependents with a warning they can't fix
-    /// (dependents have to wait for a new release of the affected crate to be
-    /// published).
-    ///
-    /// After a lint has been in this state for a while, consider graduating
-    /// it to [`FutureIncompatibilityReason::FutureReleaseErrorReportInDeps`].
-    FutureReleaseErrorDontReportInDeps,
-    /// This will be an error in a future release, and
-    /// Cargo should create a report even for dependencies
-    ///
-    /// This is the *only* reason that will make future incompatibility warnings show up in cargo's
-    /// reports. All other future incompatibility warnings are not visible when they occur in a
-    /// dependency.
+    /// future.
     ///
-    /// Choose this variant after the lint has been sitting in the
-    /// [`FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps`]
-    /// state for a while, and you feel like it is ready to graduate to
-    /// warning everyone. It is a good signal that it is ready if you can
-    /// determine that all or most affected crates on crates.io have been
-    /// updated.
+    /// After a lint has been in this state for a while and you feel like it is ready to graduate
+    /// to warning everyone, consider setting [`FutureIncompatibleInfo::report_in_deps`] to true.
+    /// (see it's documentation for more guidance)
     ///
     /// After some period of time, lints with this variant can be turned into
     /// hard errors (and the lint removed). Preferably when there is some
     /// confidence that the number of impacted projects is very small (few
     /// should have a broken dependency in their dependency tree).
-    ///
-    /// [`EditionAndFutureReleaseError`]: FutureIncompatibilityReason::EditionAndFutureReleaseError
-    FutureReleaseErrorReportInDeps,
+    FutureReleaseError,
     /// Code that changes meaning in some way in a
     /// future release.
     ///
     /// Choose this variant when the semantics of existing code is changing,
-    /// (as opposed to
-    /// [`FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps`],
+    /// (as opposed to [`FutureIncompatibilityReason::FutureReleaseError`],
     /// which is for when code is going to be rejected in the future).
     FutureReleaseSemanticsChange,
     /// Previously accepted code that will become an
@@ -454,13 +444,12 @@ pub enum FutureIncompatibilityReason {
     /// This will be an error in the provided edition *and* in a future
     /// release.
     ///
-    /// This variant a combination of [`FutureReleaseErrorDontReportInDeps`]
-    /// and [`EditionError`]. This is useful in rare cases when we
-    /// want to have "preview" of a breaking change in an edition, but do a
-    /// breaking change later on all editions anyway.
+    /// This variant a combination of [`FutureReleaseError`] and [`EditionError`].
+    /// This is useful in rare cases when we want to have "preview" of a breaking
+    /// change in an edition, but do a breaking change later on all editions anyway.
     ///
     /// [`EditionError`]: FutureIncompatibilityReason::EditionError
-    /// [`FutureReleaseErrorDontReportInDeps`]: FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps
+    /// [`FutureReleaseError`]: FutureIncompatibilityReason::FutureReleaseError
     EditionAndFutureReleaseError(Edition),
     /// This will change meaning in the provided edition *and* in a future
     /// release.
@@ -478,7 +467,7 @@ pub enum FutureIncompatibilityReason {
     /// Choose this variant if the built-in text of the diagnostic of the
     /// other variants doesn't match your situation. This is behaviorally
     /// equivalent to
-    /// [`FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps`].
+    /// [`FutureIncompatibilityReason::FutureReleaseError`].
     Custom(&'static str),
 }
 
@@ -490,34 +479,20 @@ impl FutureIncompatibilityReason {
             | Self::EditionAndFutureReleaseError(e)
             | Self::EditionAndFutureReleaseSemanticsChange(e) => Some(e),
 
-            FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps
-            | FutureIncompatibilityReason::FutureReleaseErrorReportInDeps
+            FutureIncompatibilityReason::FutureReleaseError
             | FutureIncompatibilityReason::FutureReleaseSemanticsChange
             | FutureIncompatibilityReason::Custom(_) => None,
         }
     }
-
-    pub fn has_future_breakage(self) -> bool {
-        match self {
-            FutureIncompatibilityReason::FutureReleaseErrorReportInDeps => true,
-
-            FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps
-            | FutureIncompatibilityReason::FutureReleaseSemanticsChange
-            | FutureIncompatibilityReason::EditionError(_)
-            | FutureIncompatibilityReason::EditionSemanticsChange(_)
-            | FutureIncompatibilityReason::EditionAndFutureReleaseError(_)
-            | FutureIncompatibilityReason::EditionAndFutureReleaseSemanticsChange(_)
-            | FutureIncompatibilityReason::Custom(_) => false,
-        }
-    }
 }
 
 impl FutureIncompatibleInfo {
     pub const fn default_fields_for_macro() -> Self {
         FutureIncompatibleInfo {
             reference: "",
-            reason: FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps,
+            reason: FutureIncompatibilityReason::FutureReleaseError,
             explain_reason: true,
+            report_in_deps: false,
         }
     }
 }
@@ -635,12 +610,6 @@ pub enum DeprecatedSinceKind {
     InVersion(String),
 }
 
-#[derive(Debug)]
-pub enum ElidedLifetimeResolution {
-    Static,
-    Param(Symbol, Span),
-}
-
 // This could be a closure, but then implementing derive trait
 // becomes hacky (and it gets allocated).
 #[derive(Debug)]
@@ -653,10 +622,6 @@ pub enum BuiltinLintDiag {
     },
     MacroExpandedMacroExportsAccessedByAbsolutePaths(Span),
     ElidedLifetimesInPaths(usize, Span, bool, Span),
-    ElidedNamedLifetimes {
-        elided: (Span, MissingLifetimeKind),
-        resolution: ElidedLifetimeResolution,
-    },
     UnknownCrateTypes {
         span: Span,
         candidate: Option<Symbol>,
@@ -698,6 +663,14 @@ pub enum BuiltinLintDiag {
         is_string: bool,
         suggestion: Span,
     },
+    HiddenUnicodeCodepoints {
+        label: String,
+        count: usize,
+        span_label: Span,
+        labels: Option<Vec<(char, Span)>>,
+        escape: bool,
+        spans: Vec<(char, Span)>,
+    },
     TrailingMacro(bool, Ident),
     BreakWithLabelAndLoop(Span),
     UnicodeTextFlow(Span, String),
@@ -736,6 +709,7 @@ pub enum BuiltinLintDiag {
         ty: String,
     },
     UnusedExternCrate {
+        span: Span,
         removal_span: Span,
     },
     ExternCrateNotIdiomatic {
diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs
index a662694ac38..9a6549379d3 100644
--- a/compiler/rustc_llvm/build.rs
+++ b/compiler/rustc_llvm/build.rs
@@ -228,10 +228,10 @@ fn main() {
     let mut cmd = Command::new(&llvm_config);
     cmd.arg(llvm_link_arg).arg("--libs");
 
-    // Don't link system libs if cross-compiling unless targeting Windows.
+    // Don't link system libs if cross-compiling unless targeting Windows from Windows host.
     // On Windows system DLLs aren't linked directly, instead import libraries are used.
     // These import libraries are independent of the host.
-    if !is_crossed || target.contains("windows") {
+    if !is_crossed || target.contains("windows") && host.contains("windows") {
         cmd.arg("--system-libs");
     }
 
diff --git a/compiler/rustc_macros/src/lib.rs b/compiler/rustc_macros/src/lib.rs
index 81817018cb1..42d006ef301 100644
--- a/compiler/rustc_macros/src/lib.rs
+++ b/compiler/rustc_macros/src/lib.rs
@@ -3,7 +3,6 @@
 #![feature(if_let_guard)]
 #![feature(never_type)]
 #![feature(proc_macro_diagnostic)]
-#![feature(proc_macro_span)]
 #![feature(proc_macro_tracked_env)]
 // tidy-alphabetical-end
 
diff --git a/compiler/rustc_macros/src/query.rs b/compiler/rustc_macros/src/query.rs
index ee377277017..2196f71299a 100644
--- a/compiler/rustc_macros/src/query.rs
+++ b/compiler/rustc_macros/src/query.rs
@@ -51,6 +51,7 @@ impl Parse for Query {
         let key = Pat::parse_single(&arg_content)?;
         arg_content.parse::<Token![:]>()?;
         let arg = arg_content.parse()?;
+        let _ = arg_content.parse::<Option<Token![,]>>()?;
         let result = input.parse()?;
 
         // Parse the query modifiers
diff --git a/compiler/rustc_metadata/Cargo.toml b/compiler/rustc_metadata/Cargo.toml
index cfe412e99d8..a163518fd19 100644
--- a/compiler/rustc_metadata/Cargo.toml
+++ b/compiler/rustc_metadata/Cargo.toml
@@ -23,6 +23,9 @@ rustc_hir_pretty = { path = "../rustc_hir_pretty" }
 rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
+# We must use the proc_macro version that we will compile proc-macros against,
+# not the one from our own sysroot.
+rustc_proc_macro = { path = "../rustc_proc_macro" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
diff --git a/compiler/rustc_metadata/messages.ftl b/compiler/rustc_metadata/messages.ftl
index cac8f34b0fa..3bef5ca114b 100644
--- a/compiler/rustc_metadata/messages.ftl
+++ b/compiler/rustc_metadata/messages.ftl
@@ -2,7 +2,7 @@ metadata_as_needed_compatibility =
     linking modifier `as-needed` is only compatible with `dylib` and `framework` linking kinds
 
 metadata_async_drop_types_in_dependency =
-    found async drop types in dependecy `{$extern_crate}`, but async_drop feature is disabled for `{$local_crate}`
+    found async drop types in dependency `{$extern_crate}`, but async_drop feature is disabled for `{$local_crate}`
     .help = if async drop type will be dropped in a crate without `feature(async_drop)`, sync Drop will be used
 
 metadata_bad_panic_strategy =
@@ -272,6 +272,9 @@ metadata_raw_dylib_no_nul =
 metadata_raw_dylib_only_windows =
     link kind `raw-dylib` is only supported on Windows targets
 
+metadata_raw_dylib_unsupported_abi =
+    ABI not supported by `#[link(kind = "raw-dylib")]` on this architecture
+
 metadata_renaming_no_link =
     renaming of the library `{$lib_name}` was specified, however this crate contains no `#[link(...)]` attributes referencing this library
 
@@ -319,12 +322,6 @@ metadata_unknown_link_modifier =
 
 metadata_unknown_target_modifier_unsafe_allowed = unknown target modifier `{$flag_name}`, requested by `-Cunsafe-allow-abi-mismatch={$flag_name}`
 
-metadata_unsupported_abi =
-    ABI not supported by `#[link(kind = "raw-dylib")]` on this architecture
-
-metadata_unsupported_abi_i686 =
-    ABI not supported by `#[link(kind = "raw-dylib")]` on i686
-
 metadata_wasm_c_abi =
     older versions of the `wasm-bindgen` crate are incompatible with current versions of Rust; please update to `wasm-bindgen` v0.2.88
 
diff --git a/compiler/rustc_metadata/src/creader.rs b/compiler/rustc_metadata/src/creader.rs
index c7e9a2936f5..de19c10bc57 100644
--- a/compiler/rustc_metadata/src/creader.rs
+++ b/compiler/rustc_metadata/src/creader.rs
@@ -7,7 +7,6 @@ use std::str::FromStr;
 use std::time::Duration;
 use std::{cmp, env, iter};
 
-use proc_macro::bridge::client::ProcMacro;
 use rustc_ast::expand::allocator::{AllocatorKind, alloc_error_handler_name, global_fn_name};
 use rustc_ast::{self as ast, *};
 use rustc_data_structures::fx::FxHashSet;
@@ -22,7 +21,9 @@ use rustc_hir::def_id::{CrateNum, LOCAL_CRATE, LocalDefId, StableCrateId};
 use rustc_hir::definitions::Definitions;
 use rustc_index::IndexVec;
 use rustc_middle::bug;
+use rustc_middle::ty::data_structures::IndexSet;
 use rustc_middle::ty::{TyCtxt, TyCtxtFeed};
+use rustc_proc_macro::bridge::client::ProcMacro;
 use rustc_session::config::{
     self, CrateType, ExtendedTargetModifierInfo, ExternLocation, OptionsTargetModifiers,
     TargetModifier,
@@ -281,7 +282,7 @@ impl CStore {
             .filter_map(|(cnum, data)| data.as_deref_mut().map(|data| (cnum, data)))
     }
 
-    fn push_dependencies_in_postorder(&self, deps: &mut Vec<CrateNum>, cnum: CrateNum) {
+    fn push_dependencies_in_postorder(&self, deps: &mut IndexSet<CrateNum>, cnum: CrateNum) {
         if !deps.contains(&cnum) {
             let data = self.get_crate_data(cnum);
             for dep in data.dependencies() {
@@ -290,12 +291,12 @@ impl CStore {
                 }
             }
 
-            deps.push(cnum);
+            deps.insert(cnum);
         }
     }
 
-    pub(crate) fn crate_dependencies_in_postorder(&self, cnum: CrateNum) -> Vec<CrateNum> {
-        let mut deps = Vec::new();
+    pub(crate) fn crate_dependencies_in_postorder(&self, cnum: CrateNum) -> IndexSet<CrateNum> {
+        let mut deps = IndexSet::default();
         if cnum == LOCAL_CRATE {
             for (cnum, _) in self.iter_crate_data() {
                 self.push_dependencies_in_postorder(&mut deps, cnum);
@@ -306,10 +307,11 @@ impl CStore {
         deps
     }
 
-    fn crate_dependencies_in_reverse_postorder(&self, cnum: CrateNum) -> Vec<CrateNum> {
-        let mut deps = self.crate_dependencies_in_postorder(cnum);
-        deps.reverse();
-        deps
+    fn crate_dependencies_in_reverse_postorder(
+        &self,
+        cnum: CrateNum,
+    ) -> impl Iterator<Item = CrateNum> {
+        self.crate_dependencies_in_postorder(cnum).into_iter().rev()
     }
 
     pub(crate) fn injected_panic_runtime(&self) -> Option<CrateNum> {
diff --git a/compiler/rustc_metadata/src/errors.rs b/compiler/rustc_metadata/src/errors.rs
index 16f59793e63..71da4290174 100644
--- a/compiler/rustc_metadata/src/errors.rs
+++ b/compiler/rustc_metadata/src/errors.rs
@@ -300,15 +300,8 @@ pub struct NoLinkModOverride {
 }
 
 #[derive(Diagnostic)]
-#[diag(metadata_unsupported_abi_i686)]
-pub struct UnsupportedAbiI686 {
-    #[primary_span]
-    pub span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(metadata_unsupported_abi)]
-pub struct UnsupportedAbi {
+#[diag(metadata_raw_dylib_unsupported_abi)]
+pub struct RawDylibUnsupportedAbi {
     #[primary_span]
     pub span: Span,
 }
diff --git a/compiler/rustc_metadata/src/lib.rs b/compiler/rustc_metadata/src/lib.rs
index 97b67140fa2..389a4ab7466 100644
--- a/compiler/rustc_metadata/src/lib.rs
+++ b/compiler/rustc_metadata/src/lib.rs
@@ -16,8 +16,6 @@
 #![feature(trusted_len)]
 // tidy-alphabetical-end
 
-extern crate proc_macro;
-
 pub use rmeta::provide;
 
 mod dependency_format;
diff --git a/compiler/rustc_metadata/src/locator.rs b/compiler/rustc_metadata/src/locator.rs
index 10123cb9a9d..79015aab5d3 100644
--- a/compiler/rustc_metadata/src/locator.rs
+++ b/compiler/rustc_metadata/src/locator.rs
@@ -435,7 +435,7 @@ impl<'a> CrateLocator<'a> {
                         info!("lib candidate: {}", spf.path.display());
 
                         let (rlibs, rmetas, dylibs, interfaces) =
-                            candidates.entry(hash.to_string()).or_default();
+                            candidates.entry(hash).or_default();
                         {
                             // As a perforamnce optimisation we canonicalize the path and skip
                             // ones we've already seeen. This allows us to ignore crates
diff --git a/compiler/rustc_metadata/src/native_libs.rs b/compiler/rustc_metadata/src/native_libs.rs
index cee9cff0775..5cdeb8935f7 100644
--- a/compiler/rustc_metadata/src/native_libs.rs
+++ b/compiler/rustc_metadata/src/native_libs.rs
@@ -652,7 +652,13 @@ impl<'tcx> Collector<'tcx> {
     ) -> DllImport {
         let span = self.tcx.def_span(item);
 
-        // this logic is similar to `Target::adjust_abi` (in rustc_target/src/spec/mod.rs) but errors on unsupported inputs
+        // This `extern` block should have been checked for general ABI support before, but let's
+        // double-check that.
+        assert!(self.tcx.sess.target.is_abi_supported(abi));
+
+        // This logic is similar to `AbiMap::canonize_abi` (in rustc_target/src/spec/abi_map.rs) but
+        // we need more detail than those adjustments, and we can't support all ABIs that are
+        // generally supported.
         let calling_convention = if self.tcx.sess.target.arch == "x86" {
             match abi {
                 ExternAbi::C { .. } | ExternAbi::Cdecl { .. } => DllCallingConvention::C,
@@ -679,7 +685,7 @@ impl<'tcx> Collector<'tcx> {
                     DllCallingConvention::Vectorcall(self.i686_arg_list_size(item))
                 }
                 _ => {
-                    self.tcx.dcx().emit_fatal(errors::UnsupportedAbiI686 { span });
+                    self.tcx.dcx().emit_fatal(errors::RawDylibUnsupportedAbi { span });
                 }
             }
         } else {
@@ -688,7 +694,7 @@ impl<'tcx> Collector<'tcx> {
                     DllCallingConvention::C
                 }
                 _ => {
-                    self.tcx.dcx().emit_fatal(errors::UnsupportedAbi { span });
+                    self.tcx.dcx().emit_fatal(errors::RawDylibUnsupportedAbi { span });
                 }
             }
         };
diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs
index 2e4352ca532..1dae858b7ef 100644
--- a/compiler/rustc_metadata/src/rmeta/decoder.rs
+++ b/compiler/rustc_metadata/src/rmeta/decoder.rs
@@ -6,7 +6,6 @@ use std::sync::{Arc, OnceLock};
 use std::{io, iter, mem};
 
 pub(super) use cstore_impl::provide;
-use proc_macro::bridge::client::ProcMacro;
 use rustc_ast as ast;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::FxIndexMap;
@@ -26,6 +25,7 @@ use rustc_middle::mir::interpret::{AllocDecodingSession, AllocDecodingState};
 use rustc_middle::ty::Visibility;
 use rustc_middle::ty::codec::TyDecoder;
 use rustc_middle::{bug, implement_ty_decoder};
+use rustc_proc_macro::bridge::client::ProcMacro;
 use rustc_serialize::opaque::MemDecoder;
 use rustc_serialize::{Decodable, Decoder};
 use rustc_session::Session;
diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
index 97d31565733..375928c2245 100644
--- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
+++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
@@ -425,6 +425,7 @@ provide! { tcx, def_id, other, cdata,
     doc_link_traits_in_scope => {
         tcx.arena.alloc_from_iter(cdata.get_doc_link_traits_in_scope(def_id.index))
     }
+    anon_const_kind => { table }
 }
 
 pub(in crate::rmeta) fn provide(providers: &mut Providers) {
@@ -548,8 +549,9 @@ pub(in crate::rmeta) fn provide(providers: &mut Providers) {
         has_global_allocator: |tcx, LocalCrate| CStore::from_tcx(tcx).has_global_allocator(),
         has_alloc_error_handler: |tcx, LocalCrate| CStore::from_tcx(tcx).has_alloc_error_handler(),
         postorder_cnums: |tcx, ()| {
-            tcx.arena
-                .alloc_slice(&CStore::from_tcx(tcx).crate_dependencies_in_postorder(LOCAL_CRATE))
+            tcx.arena.alloc_from_iter(
+                CStore::from_tcx(tcx).crate_dependencies_in_postorder(LOCAL_CRATE).into_iter(),
+            )
         },
         crates: |tcx, ()| {
             // The list of loaded crates is now frozen in query cache,
diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs
index 7ac72ef814a..3ab989d2d3b 100644
--- a/compiler/rustc_metadata/src/rmeta/encoder.rs
+++ b/compiler/rustc_metadata/src/rmeta/encoder.rs
@@ -1569,6 +1569,9 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                         <- tcx.explicit_implied_const_bounds(def_id).skip_binder());
                 }
             }
+            if let DefKind::AnonConst = def_kind {
+                record!(self.tables.anon_const_kind[def_id] <- self.tcx.anon_const_kind(def_id));
+            }
             if tcx.impl_method_has_trait_impl_trait_tys(def_id)
                 && let Ok(table) = self.tcx.collect_return_position_impl_trait_in_trait_tys(def_id)
             {
diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs
index d3d928aa88e..077835283e9 100644
--- a/compiler/rustc_metadata/src/rmeta/mod.rs
+++ b/compiler/rustc_metadata/src/rmeta/mod.rs
@@ -480,6 +480,7 @@ define_tables! {
     doc_link_traits_in_scope: Table<DefIndex, LazyArray<DefId>>,
     assumed_wf_types_for_rpitit: Table<DefIndex, LazyArray<(Ty<'static>, Span)>>,
     opaque_ty_origin: Table<DefIndex, LazyValue<hir::OpaqueTyOrigin<DefId>>>,
+    anon_const_kind: Table<DefIndex, LazyValue<ty::AnonConstKind>>,
 }
 
 #[derive(TyEncodable, TyDecodable)]
diff --git a/compiler/rustc_middle/src/dep_graph/mod.rs b/compiler/rustc_middle/src/dep_graph/mod.rs
index 931d67087ac..0a8e6153817 100644
--- a/compiler/rustc_middle/src/dep_graph/mod.rs
+++ b/compiler/rustc_middle/src/dep_graph/mod.rs
@@ -2,6 +2,7 @@ use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_query_system::ich::StableHashingContext;
 use rustc_session::Session;
 
+use crate::ty::print::with_reduced_queries;
 use crate::ty::{self, TyCtxt};
 
 #[macro_use]
@@ -84,4 +85,8 @@ impl<'tcx> DepContext for TyCtxt<'tcx> {
     fn dep_kind_info(&self, dk: DepKind) -> &DepKindStruct<'tcx> {
         &self.query_kinds[dk.as_usize()]
     }
+
+    fn with_reduced_queries<T>(self, f: impl FnOnce() -> T) -> T {
+        with_reduced_queries!(f())
+    }
 }
diff --git a/compiler/rustc_middle/src/error.rs b/compiler/rustc_middle/src/error.rs
index bd315577efb..6c6b12fed67 100644
--- a/compiler/rustc_middle/src/error.rs
+++ b/compiler/rustc_middle/src/error.rs
@@ -95,7 +95,7 @@ pub(crate) struct StrictCoherenceNeedsNegativeCoherence {
 #[diag(middle_requires_lang_item)]
 pub(crate) struct RequiresLangItem {
     #[primary_span]
-    pub span: Option<Span>,
+    pub span: Span,
     pub name: Symbol,
 }
 
diff --git a/compiler/rustc_middle/src/hir/map.rs b/compiler/rustc_middle/src/hir/map.rs
index fee707f7b4c..3de97c8c0d9 100644
--- a/compiler/rustc_middle/src/hir/map.rs
+++ b/compiler/rustc_middle/src/hir/map.rs
@@ -310,7 +310,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// This should only be used for determining the context of a body, a return
     /// value of `Some` does not always suggest that the owner of the body is `const`,
     /// just that it has to be checked as if it were.
-    pub fn hir_body_const_context(self, def_id: impl Into<DefId>) -> Option<ConstContext> {
+    pub fn hir_body_const_context(self, def_id: LocalDefId) -> Option<ConstContext> {
         let def_id = def_id.into();
         let ccx = match self.hir_body_owner_kind(def_id) {
             BodyOwnerKind::Const { inline } => ConstContext::Const { inline },
@@ -920,7 +920,7 @@ impl<'tcx> TyCtxt<'tcx> {
             }) => until_within(*outer_span, generics.where_clause_span),
             // Constants and Statics.
             Node::Item(Item {
-                kind: ItemKind::Const(_, ty, ..) | ItemKind::Static(_, ty, ..),
+                kind: ItemKind::Const(_, _, ty, _) | ItemKind::Static(_, _, ty, _),
                 span: outer_span,
                 ..
             })
diff --git a/compiler/rustc_middle/src/hir/mod.rs b/compiler/rustc_middle/src/hir/mod.rs
index a28dcb0cb8e..d1f5caaafb2 100644
--- a/compiler/rustc_middle/src/hir/mod.rs
+++ b/compiler/rustc_middle/src/hir/mod.rs
@@ -71,6 +71,7 @@ impl ModuleItems {
         self.opaques.iter().copied()
     }
 
+    /// Closures and inline consts
     pub fn nested_bodies(&self) -> impl Iterator<Item = LocalDefId> {
         self.nested_bodies.iter().copied()
     }
@@ -79,6 +80,14 @@ impl ModuleItems {
         self.owners().map(|id| id.def_id)
     }
 
+    /// Closures and inline consts
+    pub fn par_nested_bodies(
+        &self,
+        f: impl Fn(LocalDefId) -> Result<(), ErrorGuaranteed> + DynSend + DynSync,
+    ) -> Result<(), ErrorGuaranteed> {
+        try_par_for_each_in(&self.nested_bodies[..], |&&id| f(id))
+    }
+
     pub fn par_items(
         &self,
         f: impl Fn(ItemId) -> Result<(), ErrorGuaranteed> + DynSend + DynSync,
diff --git a/compiler/rustc_middle/src/hir/place.rs b/compiler/rustc_middle/src/hir/place.rs
index c3d10615cf1..a34a3419d68 100644
--- a/compiler/rustc_middle/src/hir/place.rs
+++ b/compiler/rustc_middle/src/hir/place.rs
@@ -43,6 +43,9 @@ pub enum ProjectionKind {
     ///
     /// This is unused if `-Znext-solver` is enabled.
     OpaqueCast,
+
+    /// `unwrap_binder!(expr)`
+    UnwrapUnsafeBinder,
 }
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, TyEncodable, TyDecodable, HashStable)]
diff --git a/compiler/rustc_middle/src/infer/canonical.rs b/compiler/rustc_middle/src/infer/canonical.rs
index 5b860374496..2bbc48b633c 100644
--- a/compiler/rustc_middle/src/infer/canonical.rs
+++ b/compiler/rustc_middle/src/infer/canonical.rs
@@ -27,7 +27,7 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::Lock;
 use rustc_macros::{HashStable, TypeFoldable, TypeVisitable};
 pub use rustc_type_ir as ir;
-pub use rustc_type_ir::{CanonicalTyVarKind, CanonicalVarKind};
+pub use rustc_type_ir::CanonicalTyVarKind;
 use smallvec::SmallVec;
 
 use crate::mir::ConstraintCategory;
@@ -35,9 +35,9 @@ use crate::ty::{self, GenericArg, List, Ty, TyCtxt, TypeFlags, TypeVisitableExt}
 
 pub type CanonicalQueryInput<'tcx, V> = ir::CanonicalQueryInput<TyCtxt<'tcx>, V>;
 pub type Canonical<'tcx, V> = ir::Canonical<TyCtxt<'tcx>, V>;
-pub type CanonicalVarInfo<'tcx> = ir::CanonicalVarInfo<TyCtxt<'tcx>>;
+pub type CanonicalVarKind<'tcx> = ir::CanonicalVarKind<TyCtxt<'tcx>>;
 pub type CanonicalVarValues<'tcx> = ir::CanonicalVarValues<TyCtxt<'tcx>>;
-pub type CanonicalVarInfos<'tcx> = &'tcx List<CanonicalVarInfo<'tcx>>;
+pub type CanonicalVarKinds<'tcx> = &'tcx List<CanonicalVarKind<'tcx>>;
 
 /// When we canonicalize a value to form a query, we wind up replacing
 /// various parts of it with canonical variables. This struct stores
diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs
index cb3fdd4d3f7..667361b3ca0 100644
--- a/compiler/rustc_middle/src/lib.rs
+++ b/compiler/rustc_middle/src/lib.rs
@@ -51,9 +51,9 @@
 #![feature(negative_impls)]
 #![feature(never_type)]
 #![feature(ptr_alignment_type)]
+#![feature(round_char_boundary)]
 #![feature(rustc_attrs)]
 #![feature(rustdoc_internals)]
-#![feature(trusted_len)]
 #![feature(try_blocks)]
 #![feature(try_trait_v2)]
 #![feature(try_trait_v2_yeet)]
diff --git a/compiler/rustc_middle/src/lint.rs b/compiler/rustc_middle/src/lint.rs
index d5a408fdfa6..341a735f88f 100644
--- a/compiler/rustc_middle/src/lint.rs
+++ b/compiler/rustc_middle/src/lint.rs
@@ -299,7 +299,7 @@ pub fn lint_level(
         let has_future_breakage = future_incompatible.map_or(
             // Default allow lints trigger too often for testing.
             sess.opts.unstable_opts.future_incompat_test && lint.default_level != Level::Allow,
-            |incompat| incompat.reason.has_future_breakage(),
+            |incompat| incompat.report_in_deps,
         );
 
         // Convert lint level to error level.
@@ -370,8 +370,7 @@ pub fn lint_level(
 
         if let Some(future_incompatible) = future_incompatible {
             let explanation = match future_incompatible.reason {
-                FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps
-                | FutureIncompatibilityReason::FutureReleaseErrorReportInDeps => {
+                FutureIncompatibilityReason::FutureReleaseError => {
                     "this was previously accepted by the compiler but is being phased out; \
                          it will become a hard error in a future release!"
                         .to_owned()
diff --git a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
index 00da1a6aeec..f21cf5fa45e 100644
--- a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
+++ b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
@@ -96,49 +96,46 @@ bitflags::bitflags! {
         /// `#[cold]`: a hint to LLVM that this function, when called, is never on
         /// the hot path.
         const COLD                      = 1 << 0;
-        /// `#[rustc_allocator]`: a hint to LLVM that the pointer returned from this
-        /// function is never null and the function has no side effects other than allocating.
-        const ALLOCATOR                 = 1 << 1;
-        /// An indicator that function will never unwind. Will become obsolete
-        /// once C-unwind is fully stabilized.
-        const NEVER_UNWIND              = 1 << 3;
+        /// `#[rustc_nounwind]`: An indicator that function will never unwind.
+        const NEVER_UNWIND              = 1 << 1;
         /// `#[naked]`: an indicator to LLVM that no function prologue/epilogue
         /// should be generated.
-        const NAKED                     = 1 << 4;
+        const NAKED                     = 1 << 2;
         /// `#[no_mangle]`: an indicator that the function's name should be the same
         /// as its symbol.
-        const NO_MANGLE                 = 1 << 5;
+        const NO_MANGLE                 = 1 << 3;
         /// `#[rustc_std_internal_symbol]`: an indicator that this symbol is a
         /// "weird symbol" for the standard library in that it has slightly
         /// different linkage, visibility, and reachability rules.
-        const RUSTC_STD_INTERNAL_SYMBOL = 1 << 6;
+        const RUSTC_STD_INTERNAL_SYMBOL = 1 << 4;
         /// `#[thread_local]`: indicates a static is actually a thread local
         /// piece of memory
-        const THREAD_LOCAL              = 1 << 8;
-        /// `#[used]`: indicates that LLVM can't eliminate this function (but the
+        const THREAD_LOCAL              = 1 << 5;
+        /// `#[used(compiler)]`: indicates that LLVM can't eliminate this function (but the
         /// linker can!).
-        const USED                      = 1 << 9;
+        const USED_COMPILER             = 1 << 6;
+        /// `#[used(linker)]`:
+        /// indicates that neither LLVM nor the linker will eliminate this function.
+        const USED_LINKER               = 1 << 7;
         /// `#[track_caller]`: allow access to the caller location
-        const TRACK_CALLER              = 1 << 10;
+        const TRACK_CALLER              = 1 << 8;
         /// #[ffi_pure]: applies clang's `pure` attribute to a foreign function
         /// declaration.
-        const FFI_PURE                  = 1 << 11;
+        const FFI_PURE                  = 1 << 9;
         /// #[ffi_const]: applies clang's `const` attribute to a foreign function
         /// declaration.
-        const FFI_CONST                 = 1 << 12;
-        // (Bit 13 was used for `#[cmse_nonsecure_entry]`, but is now unused.)
-        // (Bit 14 was used for `#[coverage(off)]`, but is now unused.)
-        /// `#[used(linker)]`:
-        /// indicates that neither LLVM nor the linker will eliminate this function.
-        const USED_LINKER               = 1 << 15;
+        const FFI_CONST                 = 1 << 10;
+        /// `#[rustc_allocator]`: a hint to LLVM that the pointer returned from this
+        /// function is never null and the function has no side effects other than allocating.
+        const ALLOCATOR                 = 1 << 11;
         /// `#[rustc_deallocator]`: a hint to LLVM that the function only deallocates memory.
-        const DEALLOCATOR               = 1 << 16;
+        const DEALLOCATOR               = 1 << 12;
         /// `#[rustc_reallocator]`: a hint to LLVM that the function only reallocates memory.
-        const REALLOCATOR               = 1 << 17;
+        const REALLOCATOR               = 1 << 13;
         /// `#[rustc_allocator_zeroed]`: a hint to LLVM that the function only allocates zeroed memory.
-        const ALLOCATOR_ZEROED          = 1 << 18;
+        const ALLOCATOR_ZEROED          = 1 << 14;
         /// `#[no_builtins]`: indicates that disable implicit builtin knowledge of functions for the function.
-        const NO_BUILTINS               = 1 << 19;
+        const NO_BUILTINS               = 1 << 15;
     }
 }
 rustc_data_structures::external_bitflags_debug! { CodegenFnAttrFlags }
diff --git a/compiler/rustc_middle/src/middle/lang_items.rs b/compiler/rustc_middle/src/middle/lang_items.rs
index 0f92c1910f1..93264f02cc2 100644
--- a/compiler/rustc_middle/src/middle/lang_items.rs
+++ b/compiler/rustc_middle/src/middle/lang_items.rs
@@ -17,7 +17,7 @@ use crate::ty::{self, TyCtxt};
 impl<'tcx> TyCtxt<'tcx> {
     /// Returns the `DefId` for a given `LangItem`.
     /// If not found, fatally aborts compilation.
-    pub fn require_lang_item(self, lang_item: LangItem, span: Option<Span>) -> DefId {
+    pub fn require_lang_item(self, lang_item: LangItem, span: Span) -> DefId {
         self.lang_items().get(lang_item).unwrap_or_else(|| {
             self.dcx().emit_fatal(crate::error::RequiresLangItem { span, name: lang_item.name() });
         })
diff --git a/compiler/rustc_middle/src/mir/interpret/allocation.rs b/compiler/rustc_middle/src/mir/interpret/allocation.rs
index 57aafbb26bc..dd55d039794 100644
--- a/compiler/rustc_middle/src/mir/interpret/allocation.rs
+++ b/compiler/rustc_middle/src/mir/interpret/allocation.rs
@@ -27,12 +27,21 @@ use crate::ty;
 
 /// Functionality required for the bytes of an `Allocation`.
 pub trait AllocBytes: Clone + fmt::Debug + Deref<Target = [u8]> + DerefMut<Target = [u8]> {
+    /// The type of extra parameters passed in when creating an allocation.
+    /// Can be used by `interpret::Machine` instances to make runtime-configuration-dependent
+    /// decisions about the allocation strategy.
+    type AllocParams;
+
     /// Create an `AllocBytes` from a slice of `u8`.
-    fn from_bytes<'a>(slice: impl Into<Cow<'a, [u8]>>, _align: Align) -> Self;
+    fn from_bytes<'a>(
+        slice: impl Into<Cow<'a, [u8]>>,
+        _align: Align,
+        _params: Self::AllocParams,
+    ) -> Self;
 
     /// Create a zeroed `AllocBytes` of the specified size and alignment.
     /// Returns `None` if we ran out of memory on the host.
-    fn zeroed(size: Size, _align: Align) -> Option<Self>;
+    fn zeroed(size: Size, _align: Align, _params: Self::AllocParams) -> Option<Self>;
 
     /// Gives direct access to the raw underlying storage.
     ///
@@ -51,11 +60,13 @@ pub trait AllocBytes: Clone + fmt::Debug + Deref<Target = [u8]> + DerefMut<Targe
 
 /// Default `bytes` for `Allocation` is a `Box<u8>`.
 impl AllocBytes for Box<[u8]> {
-    fn from_bytes<'a>(slice: impl Into<Cow<'a, [u8]>>, _align: Align) -> Self {
+    type AllocParams = ();
+
+    fn from_bytes<'a>(slice: impl Into<Cow<'a, [u8]>>, _align: Align, _params: ()) -> Self {
         Box::<[u8]>::from(slice.into())
     }
 
-    fn zeroed(size: Size, _align: Align) -> Option<Self> {
+    fn zeroed(size: Size, _align: Align, _params: ()) -> Option<Self> {
         let bytes = Box::<[u8]>::try_new_zeroed_slice(size.bytes().try_into().ok()?).ok()?;
         // SAFETY: the box was zero-allocated, which is a valid initial value for Box<[u8]>
         let bytes = unsafe { bytes.assume_init() };
@@ -172,9 +183,8 @@ fn all_zero(buf: &[u8]) -> bool {
 }
 
 /// Custom encoder for [`Allocation`] to more efficiently represent the case where all bytes are 0.
-impl<Prov: Provenance, Extra, Bytes, E: Encoder> Encodable<E> for Allocation<Prov, Extra, Bytes>
+impl<Prov: Provenance, Extra, E: Encoder> Encodable<E> for Allocation<Prov, Extra, Box<[u8]>>
 where
-    Bytes: AllocBytes,
     ProvenanceMap<Prov>: Encodable<E>,
     Extra: Encodable<E>,
 {
@@ -192,9 +202,8 @@ where
     }
 }
 
-impl<Prov: Provenance, Extra, Bytes, D: Decoder> Decodable<D> for Allocation<Prov, Extra, Bytes>
+impl<Prov: Provenance, Extra, D: Decoder> Decodable<D> for Allocation<Prov, Extra, Box<[u8]>>
 where
-    Bytes: AllocBytes,
     ProvenanceMap<Prov>: Decodable<D>,
     Extra: Decodable<D>,
 {
@@ -203,7 +212,7 @@ where
 
         let len = decoder.read_usize();
         let bytes = if all_zero { vec![0u8; len] } else { decoder.read_raw_bytes(len).to_vec() };
-        let bytes = Bytes::from_bytes(bytes, align);
+        let bytes = <Box<[u8]> as AllocBytes>::from_bytes(bytes, align, ());
 
         let provenance = Decodable::decode(decoder);
         let init_mask = Decodable::decode(decoder);
@@ -395,8 +404,9 @@ impl<Prov: Provenance, Bytes: AllocBytes> Allocation<Prov, (), Bytes> {
         slice: impl Into<Cow<'a, [u8]>>,
         align: Align,
         mutability: Mutability,
+        params: <Bytes as AllocBytes>::AllocParams,
     ) -> Self {
-        let bytes = Bytes::from_bytes(slice, align);
+        let bytes = Bytes::from_bytes(slice, align, params);
         let size = Size::from_bytes(bytes.len());
         Self {
             bytes,
@@ -408,14 +418,18 @@ impl<Prov: Provenance, Bytes: AllocBytes> Allocation<Prov, (), Bytes> {
         }
     }
 
-    pub fn from_bytes_byte_aligned_immutable<'a>(slice: impl Into<Cow<'a, [u8]>>) -> Self {
-        Allocation::from_bytes(slice, Align::ONE, Mutability::Not)
+    pub fn from_bytes_byte_aligned_immutable<'a>(
+        slice: impl Into<Cow<'a, [u8]>>,
+        params: <Bytes as AllocBytes>::AllocParams,
+    ) -> Self {
+        Allocation::from_bytes(slice, Align::ONE, Mutability::Not, params)
     }
 
     fn new_inner<R>(
         size: Size,
         align: Align,
         init: AllocInit,
+        params: <Bytes as AllocBytes>::AllocParams,
         fail: impl FnOnce() -> R,
     ) -> Result<Self, R> {
         // We raise an error if we cannot create the allocation on the host.
@@ -424,7 +438,7 @@ impl<Prov: Provenance, Bytes: AllocBytes> Allocation<Prov, (), Bytes> {
         // deterministic. However, we can be non-deterministic here because all uses of const
         // evaluation (including ConstProp!) will make compilation fail (via hard error
         // or ICE) upon encountering a `MemoryExhausted` error.
-        let bytes = Bytes::zeroed(size, align).ok_or_else(fail)?;
+        let bytes = Bytes::zeroed(size, align, params).ok_or_else(fail)?;
 
         Ok(Allocation {
             bytes,
@@ -444,8 +458,13 @@ impl<Prov: Provenance, Bytes: AllocBytes> Allocation<Prov, (), Bytes> {
 
     /// Try to create an Allocation of `size` bytes, failing if there is not enough memory
     /// available to the compiler to do so.
-    pub fn try_new<'tcx>(size: Size, align: Align, init: AllocInit) -> InterpResult<'tcx, Self> {
-        Self::new_inner(size, align, init, || {
+    pub fn try_new<'tcx>(
+        size: Size,
+        align: Align,
+        init: AllocInit,
+        params: <Bytes as AllocBytes>::AllocParams,
+    ) -> InterpResult<'tcx, Self> {
+        Self::new_inner(size, align, init, params, || {
             ty::tls::with(|tcx| tcx.dcx().delayed_bug("exhausted memory during interpretation"));
             InterpErrorKind::ResourceExhaustion(ResourceExhaustionInfo::MemoryExhausted)
         })
@@ -457,8 +476,13 @@ impl<Prov: Provenance, Bytes: AllocBytes> Allocation<Prov, (), Bytes> {
     ///
     /// Example use case: To obtain an Allocation filled with specific data,
     /// first call this function and then call write_scalar to fill in the right data.
-    pub fn new(size: Size, align: Align, init: AllocInit) -> Self {
-        match Self::new_inner(size, align, init, || {
+    pub fn new(
+        size: Size,
+        align: Align,
+        init: AllocInit,
+        params: <Bytes as AllocBytes>::AllocParams,
+    ) -> Self {
+        match Self::new_inner(size, align, init, params, || {
             panic!(
                 "interpreter ran out of memory: cannot create allocation of {} bytes",
                 size.bytes()
@@ -488,7 +512,7 @@ impl Allocation {
     pub fn adjust_from_tcx<'tcx, Prov: Provenance, Bytes: AllocBytes>(
         &self,
         cx: &impl HasDataLayout,
-        mut alloc_bytes: impl FnMut(&[u8], Align) -> InterpResult<'tcx, Bytes>,
+        alloc_bytes: impl FnOnce(&[u8], Align) -> InterpResult<'tcx, Bytes>,
         mut adjust_ptr: impl FnMut(Pointer<CtfeProvenance>) -> InterpResult<'tcx, Pointer<Prov>>,
     ) -> InterpResult<'tcx, Allocation<Prov, (), Bytes>> {
         // Copy the data.
diff --git a/compiler/rustc_middle/src/mir/interpret/queries.rs b/compiler/rustc_middle/src/mir/interpret/queries.rs
index 4a5c42c721c..97db45a70d7 100644
--- a/compiler/rustc_middle/src/mir/interpret/queries.rs
+++ b/compiler/rustc_middle/src/mir/interpret/queries.rs
@@ -9,7 +9,6 @@ use super::{
     ReportedErrorInfo,
 };
 use crate::mir;
-use crate::query::TyCtxtEnsureOk;
 use crate::ty::{self, GenericArgs, TyCtxt, TypeVisitableExt};
 
 impl<'tcx> TyCtxt<'tcx> {
@@ -197,24 +196,3 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 }
-
-impl<'tcx> TyCtxtEnsureOk<'tcx> {
-    /// Evaluates a constant without providing any generic parameters. This is useful to evaluate consts
-    /// that can't take any generic arguments like const items or enum discriminants. If a
-    /// generic parameter is used within the constant `ErrorHandled::TooGeneric` will be returned.
-    #[instrument(skip(self), level = "debug")]
-    pub fn const_eval_poly(self, def_id: DefId) {
-        // In some situations def_id will have generic parameters within scope, but they aren't allowed
-        // to be used. So we can't use `Instance::mono`, instead we feed unresolved generic parameters
-        // into `const_eval` which will return `ErrorHandled::TooGeneric` if any of them are
-        // encountered.
-        let args = GenericArgs::identity_for_item(self.tcx, def_id);
-        let instance = ty::Instance::new_raw(def_id, self.tcx.erase_regions(args));
-        let cid = GlobalId { instance, promoted: None };
-        let typing_env = ty::TypingEnv::post_analysis(self.tcx, def_id);
-        // Const-eval shouldn't depend on lifetimes at all, so we can erase them, which should
-        // improve caching of queries.
-        let inputs = self.tcx.erase_regions(typing_env.as_query_input(cid));
-        self.eval_to_const_value_raw(inputs)
-    }
-}
diff --git a/compiler/rustc_middle/src/mir/interpret/value.rs b/compiler/rustc_middle/src/mir/interpret/value.rs
index 9d462093b9e..7ba0e5b5e07 100644
--- a/compiler/rustc_middle/src/mir/interpret/value.rs
+++ b/compiler/rustc_middle/src/mir/interpret/value.rs
@@ -180,27 +180,27 @@ impl<Prov> Scalar<Prov> {
 
     #[inline]
     pub fn from_i8(i: i8) -> Self {
-        Self::from_int(i, Size::from_bits(8))
+        Self::Int(i.into())
     }
 
     #[inline]
     pub fn from_i16(i: i16) -> Self {
-        Self::from_int(i, Size::from_bits(16))
+        Self::Int(i.into())
     }
 
     #[inline]
     pub fn from_i32(i: i32) -> Self {
-        Self::from_int(i, Size::from_bits(32))
+        Self::Int(i.into())
     }
 
     #[inline]
     pub fn from_i64(i: i64) -> Self {
-        Self::from_int(i, Size::from_bits(64))
+        Self::Int(i.into())
     }
 
     #[inline]
     pub fn from_i128(i: i128) -> Self {
-        Self::from_int(i, Size::from_bits(128))
+        Self::Int(i.into())
     }
 
     #[inline]
diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs
index 7243f87ee63..47ba850d50d 100644
--- a/compiler/rustc_middle/src/mir/mono.rs
+++ b/compiler/rustc_middle/src/mir/mono.rs
@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::fmt;
 use std::hash::Hash;
 
@@ -468,6 +469,29 @@ impl<'tcx> CodegenUnit<'tcx> {
         hash.as_u128().to_base_fixed_len(CASE_INSENSITIVE)
     }
 
+    pub fn shorten_name(human_readable_name: &str) -> Cow<'_, str> {
+        // Set a limit a somewhat below the common platform limits for file names.
+        const MAX_CGU_NAME_LENGTH: usize = 200;
+        const TRUNCATED_NAME_PREFIX: &str = "-trunc-";
+        if human_readable_name.len() > MAX_CGU_NAME_LENGTH {
+            let mangled_name = Self::mangle_name(human_readable_name);
+            // Determine a safe byte offset to truncate the name to
+            let truncate_to = human_readable_name.floor_char_boundary(
+                MAX_CGU_NAME_LENGTH - TRUNCATED_NAME_PREFIX.len() - mangled_name.len(),
+            );
+            format!(
+                "{}{}{}",
+                &human_readable_name[..truncate_to],
+                TRUNCATED_NAME_PREFIX,
+                mangled_name
+            )
+            .into()
+        } else {
+            // If the name is short enough, we can just return it as is.
+            human_readable_name.into()
+        }
+    }
+
     pub fn compute_size_estimate(&mut self) {
         // The size of a codegen unit as the sum of the sizes of the items
         // within it.
@@ -604,7 +628,7 @@ impl<'tcx> CodegenUnitNameBuilder<'tcx> {
         let cgu_name = self.build_cgu_name_no_mangle(cnum, components, special_suffix);
 
         if self.tcx.sess.opts.unstable_opts.human_readable_cgu_names {
-            cgu_name
+            Symbol::intern(&CodegenUnit::shorten_name(cgu_name.as_str()))
         } else {
             Symbol::intern(&CodegenUnit::mangle_name(cgu_name.as_str()))
         }
diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs
index 57ae7dc55c5..6b262a27500 100644
--- a/compiler/rustc_middle/src/mir/pretty.rs
+++ b/compiler/rustc_middle/src/mir/pretty.rs
@@ -1624,7 +1624,11 @@ pub fn write_allocations<'tcx>(
             Some(GlobalAlloc::Static(did)) if !tcx.is_foreign_item(did) => {
                 write!(w, " (static: {}", tcx.def_path_str(did))?;
                 if body.phase <= MirPhase::Runtime(RuntimePhase::PostCleanup)
-                    && tcx.hir_body_const_context(body.source.def_id()).is_some()
+                    && body
+                        .source
+                        .def_id()
+                        .as_local()
+                        .is_some_and(|def_id| tcx.hir_body_const_context(def_id).is_some())
                 {
                     // Statics may be cyclic and evaluating them too early
                     // in the MIR pipeline may cause cycle errors even though
diff --git a/compiler/rustc_middle/src/mir/statement.rs b/compiler/rustc_middle/src/mir/statement.rs
index d59b6df44ed..d98b40f0fcf 100644
--- a/compiler/rustc_middle/src/mir/statement.rs
+++ b/compiler/rustc_middle/src/mir/statement.rs
@@ -88,26 +88,31 @@ impl<'tcx> PlaceTy<'tcx> {
     ///
     /// Note that the resulting type has not been normalized.
     #[instrument(level = "debug", skip(tcx), ret)]
-    pub fn field_ty(self, tcx: TyCtxt<'tcx>, f: FieldIdx) -> Ty<'tcx> {
-        if let Some(variant_index) = self.variant_index {
-            match *self.ty.kind() {
+    pub fn field_ty(
+        tcx: TyCtxt<'tcx>,
+        self_ty: Ty<'tcx>,
+        variant_idx: Option<VariantIdx>,
+        f: FieldIdx,
+    ) -> Ty<'tcx> {
+        if let Some(variant_index) = variant_idx {
+            match *self_ty.kind() {
                 ty::Adt(adt_def, args) if adt_def.is_enum() => {
                     adt_def.variant(variant_index).fields[f].ty(tcx, args)
                 }
                 ty::Coroutine(def_id, args) => {
                     let mut variants = args.as_coroutine().state_tys(def_id, tcx);
                     let Some(mut variant) = variants.nth(variant_index.into()) else {
-                        bug!("variant {variant_index:?} of coroutine out of range: {self:?}");
+                        bug!("variant {variant_index:?} of coroutine out of range: {self_ty:?}");
                     };
 
-                    variant
-                        .nth(f.index())
-                        .unwrap_or_else(|| bug!("field {f:?} out of range: {self:?}"))
+                    variant.nth(f.index()).unwrap_or_else(|| {
+                        bug!("field {f:?} out of range of variant: {self_ty:?} {variant_idx:?}")
+                    })
                 }
-                _ => bug!("can't downcast non-adt non-coroutine type: {self:?}"),
+                _ => bug!("can't downcast non-adt non-coroutine type: {self_ty:?}"),
             }
         } else {
-            match self.ty.kind() {
+            match self_ty.kind() {
                 ty::Adt(adt_def, args) if !adt_def.is_enum() => {
                     adt_def.non_enum_variant().fields[f].ty(tcx, args)
                 }
@@ -116,26 +121,25 @@ impl<'tcx> PlaceTy<'tcx> {
                     .upvar_tys()
                     .get(f.index())
                     .copied()
-                    .unwrap_or_else(|| bug!("field {f:?} out of range: {self:?}")),
+                    .unwrap_or_else(|| bug!("field {f:?} out of range: {self_ty:?}")),
                 ty::CoroutineClosure(_, args) => args
                     .as_coroutine_closure()
                     .upvar_tys()
                     .get(f.index())
                     .copied()
-                    .unwrap_or_else(|| bug!("field {f:?} out of range: {self:?}")),
+                    .unwrap_or_else(|| bug!("field {f:?} out of range: {self_ty:?}")),
                 // Only prefix fields (upvars and current state) are
                 // accessible without a variant index.
-                ty::Coroutine(_, args) => args
-                    .as_coroutine()
-                    .prefix_tys()
-                    .get(f.index())
-                    .copied()
-                    .unwrap_or_else(|| bug!("field {f:?} out of range: {self:?}")),
+                ty::Coroutine(_, args) => {
+                    args.as_coroutine().prefix_tys().get(f.index()).copied().unwrap_or_else(|| {
+                        bug!("field {f:?} out of range of prefixes for {self_ty}")
+                    })
+                }
                 ty::Tuple(tys) => tys
                     .get(f.index())
                     .copied()
-                    .unwrap_or_else(|| bug!("field {f:?} out of range: {self:?}")),
-                _ => bug!("can't project out of {self:?}"),
+                    .unwrap_or_else(|| bug!("field {f:?} out of range: {self_ty:?}")),
+                _ => bug!("can't project out of {self_ty:?}"),
             }
         }
     }
@@ -148,11 +152,11 @@ impl<'tcx> PlaceTy<'tcx> {
         elems.iter().fold(self, |place_ty, &elem| place_ty.projection_ty(tcx, elem))
     }
 
-    /// Convenience wrapper around `projection_ty_core` for
-    /// `PlaceElem`, where we can just use the `Ty` that is already
-    /// stored inline on field projection elems.
+    /// Convenience wrapper around `projection_ty_core` for `PlaceElem`,
+    /// where we can just use the `Ty` that is already stored inline on
+    /// field projection elems.
     pub fn projection_ty(self, tcx: TyCtxt<'tcx>, elem: PlaceElem<'tcx>) -> PlaceTy<'tcx> {
-        self.projection_ty_core(tcx, &elem, |_, _, ty| ty, |_, ty| ty)
+        self.projection_ty_core(tcx, &elem, |ty| ty, |_, _, _, ty| ty, |ty| ty)
     }
 
     /// `place_ty.projection_ty_core(tcx, elem, |...| { ... })`
@@ -164,8 +168,9 @@ impl<'tcx> PlaceTy<'tcx> {
         self,
         tcx: TyCtxt<'tcx>,
         elem: &ProjectionElem<V, T>,
-        mut handle_field: impl FnMut(&Self, FieldIdx, T) -> Ty<'tcx>,
-        mut handle_opaque_cast_and_subtype: impl FnMut(&Self, T) -> Ty<'tcx>,
+        mut structurally_normalize: impl FnMut(Ty<'tcx>) -> Ty<'tcx>,
+        mut handle_field: impl FnMut(Ty<'tcx>, Option<VariantIdx>, FieldIdx, T) -> Ty<'tcx>,
+        mut handle_opaque_cast_and_subtype: impl FnMut(T) -> Ty<'tcx>,
     ) -> PlaceTy<'tcx>
     where
         V: ::std::fmt::Debug,
@@ -176,16 +181,16 @@ impl<'tcx> PlaceTy<'tcx> {
         }
         let answer = match *elem {
             ProjectionElem::Deref => {
-                let ty = self.ty.builtin_deref(true).unwrap_or_else(|| {
+                let ty = structurally_normalize(self.ty).builtin_deref(true).unwrap_or_else(|| {
                     bug!("deref projection of non-dereferenceable ty {:?}", self)
                 });
                 PlaceTy::from_ty(ty)
             }
             ProjectionElem::Index(_) | ProjectionElem::ConstantIndex { .. } => {
-                PlaceTy::from_ty(self.ty.builtin_index().unwrap())
+                PlaceTy::from_ty(structurally_normalize(self.ty).builtin_index().unwrap())
             }
             ProjectionElem::Subslice { from, to, from_end } => {
-                PlaceTy::from_ty(match self.ty.kind() {
+                PlaceTy::from_ty(match structurally_normalize(self.ty).kind() {
                     ty::Slice(..) => self.ty,
                     ty::Array(inner, _) if !from_end => Ty::new_array(tcx, *inner, to - from),
                     ty::Array(inner, size) if from_end => {
@@ -201,17 +206,18 @@ impl<'tcx> PlaceTy<'tcx> {
             ProjectionElem::Downcast(_name, index) => {
                 PlaceTy { ty: self.ty, variant_index: Some(index) }
             }
-            ProjectionElem::Field(f, fty) => PlaceTy::from_ty(handle_field(&self, f, fty)),
-            ProjectionElem::OpaqueCast(ty) => {
-                PlaceTy::from_ty(handle_opaque_cast_and_subtype(&self, ty))
-            }
-            ProjectionElem::Subtype(ty) => {
-                PlaceTy::from_ty(handle_opaque_cast_and_subtype(&self, ty))
-            }
+            ProjectionElem::Field(f, fty) => PlaceTy::from_ty(handle_field(
+                structurally_normalize(self.ty),
+                self.variant_index,
+                f,
+                fty,
+            )),
+            ProjectionElem::OpaqueCast(ty) => PlaceTy::from_ty(handle_opaque_cast_and_subtype(ty)),
+            ProjectionElem::Subtype(ty) => PlaceTy::from_ty(handle_opaque_cast_and_subtype(ty)),
 
             // FIXME(unsafe_binders): Rename `handle_opaque_cast_and_subtype` to be more general.
             ProjectionElem::UnwrapUnsafeBinder(ty) => {
-                PlaceTy::from_ty(handle_opaque_cast_and_subtype(&self, ty))
+                PlaceTy::from_ty(handle_opaque_cast_and_subtype(ty))
             }
         };
         debug!("projection_ty self: {:?} elem: {:?} yields: {:?}", self, elem, answer);
@@ -829,7 +835,7 @@ impl<'tcx> BinOp {
             &BinOp::Cmp => {
                 // these should be integer-like types of the same size.
                 assert_eq!(lhs_ty, rhs_ty);
-                tcx.ty_ordering_enum(None)
+                tcx.ty_ordering_enum(DUMMY_SP)
             }
         }
     }
diff --git a/compiler/rustc_middle/src/query/erase.rs b/compiler/rustc_middle/src/query/erase.rs
index 5bd111fa2f2..6035056baaf 100644
--- a/compiler/rustc_middle/src/query/erase.rs
+++ b/compiler/rustc_middle/src/query/erase.rs
@@ -6,7 +6,7 @@ use rustc_span::ErrorGuaranteed;
 
 use crate::query::CyclePlaceholder;
 use crate::ty::adjustment::CoerceUnsizedInfo;
-use crate::ty::{self, Ty};
+use crate::ty::{self, Ty, TyCtxt};
 use crate::{mir, traits};
 
 #[derive(Copy, Clone)]
@@ -207,6 +207,11 @@ impl EraseType for ty::Binder<'_, ty::FnSig<'_>> {
     type Result = [u8; size_of::<ty::Binder<'static, ty::FnSig<'static>>>()];
 }
 
+impl EraseType for ty::Binder<'_, ty::CoroutineWitnessTypes<TyCtxt<'_>>> {
+    type Result =
+        [u8; size_of::<ty::Binder<'static, ty::CoroutineWitnessTypes<TyCtxt<'static>>>>()];
+}
+
 impl EraseType for ty::Binder<'_, &'_ ty::List<Ty<'_>>> {
     type Result = [u8; size_of::<ty::Binder<'static, &'static ty::List<Ty<'static>>>>()];
 }
@@ -311,6 +316,7 @@ trivial! {
     rustc_middle::ty::Asyncness,
     rustc_middle::ty::AsyncDestructor,
     rustc_middle::ty::BoundVariableKind,
+    rustc_middle::ty::AnonConstKind,
     rustc_middle::ty::DeducedParamAttrs,
     rustc_middle::ty::Destructor,
     rustc_middle::ty::fast_reject::SimplifiedType,
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
index b2133fea08c..d03f01bf863 100644
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@@ -535,7 +535,8 @@ rustc_queries! {
         separate_provide_extern
     }
 
-    /// Fetch the THIR for a given body.
+    /// Fetch the THIR for a given body. The THIR body gets stolen by unsafety checking unless
+    /// `-Zno-steal-thir` is on.
     query thir_body(key: LocalDefId) -> Result<(&'tcx Steal<thir::Thir<'tcx>>, thir::ExprId), ErrorGuaranteed> {
         // Perf tests revealed that hashing THIR is inefficient (see #85729).
         no_hash
@@ -922,6 +923,12 @@ rustc_queries! {
         separate_provide_extern
     }
 
+    query coroutine_hidden_types(
+        def_id: DefId
+    ) -> ty::EarlyBinder<'tcx, ty::Binder<'tcx, ty::CoroutineWitnessTypes<TyCtxt<'tcx>>>> {
+        desc { "looking up the hidden types stored across await points in a coroutine" }
+    }
+
     /// Gets a map with the variances of every item in the local crate.
     ///
     /// <div class="warning">
@@ -1113,10 +1120,6 @@ rustc_queries! {
         desc { |tcx| "checking loops in {}", describe_as_module(key, tcx) }
     }
 
-    query check_mod_naked_functions(key: LocalModDefId) {
-        desc { |tcx| "checking naked functions in {}", describe_as_module(key, tcx) }
-    }
-
     query check_mod_privacy(key: LocalModDefId) {
         desc { |tcx| "checking privacy in {}", describe_as_module(key.to_local_def_id(), tcx) }
     }
@@ -1131,7 +1134,7 @@ rustc_queries! {
     /// their respective impl (i.e., part of the derive macro)
     query live_symbols_and_ignored_derived_traits(_: ()) -> &'tcx (
         LocalDefIdSet,
-        LocalDefIdMap<Vec<(DefId, DefId)>>
+        LocalDefIdMap<FxIndexSet<(DefId, DefId)>>
     ) {
         arena_cache
         desc { "finding live symbols in crate" }
@@ -1141,8 +1144,8 @@ rustc_queries! {
         desc { |tcx| "checking deathness of variables in {}", describe_as_module(key, tcx) }
     }
 
-    query check_mod_type_wf(key: LocalModDefId) -> Result<(), ErrorGuaranteed> {
-        desc { |tcx| "checking that types are well-formed in {}", describe_as_module(key, tcx) }
+    query check_type_wf(key: ()) -> Result<(), ErrorGuaranteed> {
+        desc { "checking that types are well-formed" }
         return_result_from_ensure_ok
     }
 
@@ -2586,6 +2589,11 @@ rustc_queries! {
         desc { "estimating codegen size of `{}`", key }
         cache_on_disk_if { true }
     }
+
+    query anon_const_kind(def_id: DefId) -> ty::AnonConstKind {
+        desc { |tcx| "looking up anon const kind of `{}`", tcx.def_path_str(def_id) }
+        separate_provide_extern
+    }
 }
 
 rustc_with_all_queries! { define_callbacks! }
diff --git a/compiler/rustc_middle/src/thir/visit.rs b/compiler/rustc_middle/src/thir/visit.rs
index f3da2a5cc8e..d8743814d79 100644
--- a/compiler/rustc_middle/src/thir/visit.rs
+++ b/compiler/rustc_middle/src/thir/visit.rs
@@ -3,6 +3,9 @@ use super::{
     Pat, PatKind, Stmt, StmtKind, Thir,
 };
 
+/// Every `walk_*` method uses deconstruction to access fields of structs and
+/// enums. This will result in a compile error if a field is added, which makes
+/// it more likely the appropriate visit call will be added for it.
 pub trait Visitor<'thir, 'tcx: 'thir>: Sized {
     fn thir(&self) -> &'thir Thir<'tcx>;
 
@@ -41,7 +44,8 @@ pub fn walk_expr<'thir, 'tcx: 'thir, V: Visitor<'thir, 'tcx>>(
     expr: &'thir Expr<'tcx>,
 ) {
     use ExprKind::*;
-    match expr.kind {
+    let Expr { kind, ty: _, temp_lifetime: _, span: _ } = expr;
+    match *kind {
         Scope { value, region_scope: _, lint_level: _ } => {
             visitor.visit_expr(&visitor.thir()[value])
         }
@@ -191,7 +195,8 @@ pub fn walk_stmt<'thir, 'tcx: 'thir, V: Visitor<'thir, 'tcx>>(
     visitor: &mut V,
     stmt: &'thir Stmt<'tcx>,
 ) {
-    match &stmt.kind {
+    let Stmt { kind } = stmt;
+    match kind {
         StmtKind::Expr { expr, scope: _ } => visitor.visit_expr(&visitor.thir()[*expr]),
         StmtKind::Let {
             initializer,
@@ -217,11 +222,13 @@ pub fn walk_block<'thir, 'tcx: 'thir, V: Visitor<'thir, 'tcx>>(
     visitor: &mut V,
     block: &'thir Block,
 ) {
-    for &stmt in &*block.stmts {
+    let Block { stmts, expr, targeted_by_break: _, region_scope: _, span: _, safety_mode: _ } =
+        block;
+    for &stmt in &*stmts {
         visitor.visit_stmt(&visitor.thir()[stmt]);
     }
-    if let Some(expr) = block.expr {
-        visitor.visit_expr(&visitor.thir()[expr]);
+    if let Some(expr) = expr {
+        visitor.visit_expr(&visitor.thir()[*expr]);
     }
 }
 
@@ -229,11 +236,12 @@ pub fn walk_arm<'thir, 'tcx: 'thir, V: Visitor<'thir, 'tcx>>(
     visitor: &mut V,
     arm: &'thir Arm<'tcx>,
 ) {
-    if let Some(expr) = arm.guard {
-        visitor.visit_expr(&visitor.thir()[expr])
+    let Arm { guard, pattern, body, lint_level: _, span: _, scope: _ } = arm;
+    if let Some(expr) = guard {
+        visitor.visit_expr(&visitor.thir()[*expr])
     }
-    visitor.visit_pat(&arm.pattern);
-    visitor.visit_expr(&visitor.thir()[arm.body]);
+    visitor.visit_pat(pattern);
+    visitor.visit_expr(&visitor.thir()[*body]);
 }
 
 pub fn walk_pat<'thir, 'tcx: 'thir, V: Visitor<'thir, 'tcx>>(
@@ -249,7 +257,8 @@ pub(crate) fn for_each_immediate_subpat<'a, 'tcx>(
     pat: &'a Pat<'tcx>,
     mut callback: impl FnMut(&'a Pat<'tcx>),
 ) {
-    match &pat.kind {
+    let Pat { kind, ty: _, span: _ } = pat;
+    match kind {
         PatKind::Missing
         | PatKind::Wild
         | PatKind::Binding { subpattern: None, .. }
diff --git a/compiler/rustc_middle/src/ty/adjustment.rs b/compiler/rustc_middle/src/ty/adjustment.rs
index a61a6c571a2..3bacdfe5ac8 100644
--- a/compiler/rustc_middle/src/ty/adjustment.rs
+++ b/compiler/rustc_middle/src/ty/adjustment.rs
@@ -128,8 +128,8 @@ impl OverloadedDeref {
     /// for this overloaded deref's mutability.
     pub fn method_call<'tcx>(&self, tcx: TyCtxt<'tcx>) -> DefId {
         let trait_def_id = match self.mutbl {
-            hir::Mutability::Not => tcx.require_lang_item(LangItem::Deref, None),
-            hir::Mutability::Mut => tcx.require_lang_item(LangItem::DerefMut, None),
+            hir::Mutability::Not => tcx.require_lang_item(LangItem::Deref, self.span),
+            hir::Mutability::Mut => tcx.require_lang_item(LangItem::DerefMut, self.span),
         };
         tcx.associated_items(trait_def_id)
             .in_definition_order()
diff --git a/compiler/rustc_middle/src/ty/closure.rs b/compiler/rustc_middle/src/ty/closure.rs
index ff9096695d4..df67bb505a6 100644
--- a/compiler/rustc_middle/src/ty/closure.rs
+++ b/compiler/rustc_middle/src/ty/closure.rs
@@ -117,6 +117,10 @@ impl<'tcx> CapturedPlace<'tcx> {
                     }
                 },
 
+                HirProjectionKind::UnwrapUnsafeBinder => {
+                    write!(&mut symbol, "__unwrap").unwrap();
+                }
+
                 // Ignore derefs for now, as they are likely caused by
                 // autoderefs that don't appear in the original code.
                 HirProjectionKind::Deref => {}
diff --git a/compiler/rustc_middle/src/ty/codec.rs b/compiler/rustc_middle/src/ty/codec.rs
index 5ff87959a80..e0f70737add 100644
--- a/compiler/rustc_middle/src/ty/codec.rs
+++ b/compiler/rustc_middle/src/ty/codec.rs
@@ -18,7 +18,7 @@ use rustc_span::source_map::Spanned;
 use rustc_span::{Span, SpanDecoder, SpanEncoder};
 
 use crate::arena::ArenaAllocatable;
-use crate::infer::canonical::{CanonicalVarInfo, CanonicalVarInfos};
+use crate::infer::canonical::{CanonicalVarKind, CanonicalVarKinds};
 use crate::mir::interpret::{AllocId, ConstAllocation, CtfeProvenance};
 use crate::mir::mono::MonoItem;
 use crate::mir::{self};
@@ -310,11 +310,11 @@ impl<'tcx, D: TyDecoder<'tcx>> Decodable<D> for ty::Region<'tcx> {
     }
 }
 
-impl<'tcx, D: TyDecoder<'tcx>> Decodable<D> for CanonicalVarInfos<'tcx> {
+impl<'tcx, D: TyDecoder<'tcx>> Decodable<D> for CanonicalVarKinds<'tcx> {
     fn decode(decoder: &mut D) -> Self {
         let len = decoder.read_usize();
         decoder.interner().mk_canonical_var_infos_from_iter(
-            (0..len).map::<CanonicalVarInfo<'tcx>, _>(|_| Decodable::decode(decoder)),
+            (0..len).map::<CanonicalVarKind<'tcx>, _>(|_| Decodable::decode(decoder)),
         )
     }
 }
diff --git a/compiler/rustc_middle/src/ty/consts.rs b/compiler/rustc_middle/src/ty/consts.rs
index dc5fe2d8f8b..455ac660412 100644
--- a/compiler/rustc_middle/src/ty/consts.rs
+++ b/compiler/rustc_middle/src/ty/consts.rs
@@ -2,7 +2,7 @@ use std::borrow::Cow;
 
 use rustc_data_structures::intern::Interned;
 use rustc_error_messages::MultiSpan;
-use rustc_macros::HashStable;
+use rustc_macros::{HashStable, TyDecodable, TyEncodable};
 use rustc_type_ir::walk::TypeWalker;
 use rustc_type_ir::{self as ir, TypeFlags, WithCachedTypeInfo};
 
@@ -259,3 +259,16 @@ impl<'tcx> Const<'tcx> {
         TypeWalker::new(self.into())
     }
 }
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, TyEncodable, TyDecodable, HashStable)]
+pub enum AnonConstKind {
+    /// `feature(generic_const_exprs)` anon consts are allowed to use arbitrary generic parameters in scope
+    GCE,
+    /// stable `min_const_generics` anon consts are not allowed to use any generic parameters
+    MCG,
+    /// anon consts used as the length of a repeat expr are syntactically allowed to use generic parameters
+    /// but must not depend on the actual instantiation. See #76200 for more information
+    RepeatExprCount,
+    /// anon consts outside of the type system, e.g. enum discriminants
+    NonTypeSystem,
+}
diff --git a/compiler/rustc_middle/src/ty/consts/int.rs b/compiler/rustc_middle/src/ty/consts/int.rs
index 9f5e31d894c..b087ae25486 100644
--- a/compiler/rustc_middle/src/ty/consts/int.rs
+++ b/compiler/rustc_middle/src/ty/consts/int.rs
@@ -26,6 +26,19 @@ impl ConstInt {
     }
 }
 
+/// An enum to represent the compiler-side view of `intrinsics::AtomicOrdering`.
+/// This lives here because there's a method in this file that needs it and it is entirely unclear
+/// where else to put this...
+#[derive(Debug, Copy, Clone)]
+pub enum AtomicOrdering {
+    // These values must match `intrinsics::AtomicOrdering`!
+    Relaxed = 0,
+    Release = 1,
+    Acquire = 2,
+    AcqRel = 3,
+    SeqCst = 4,
+}
+
 impl std::fmt::Debug for ConstInt {
     fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         let Self { int, signed, is_ptr_sized_integral } = *self;
@@ -318,6 +331,25 @@ impl ScalarInt {
         self.to_uint(tcx.data_layout.pointer_size).try_into().unwrap()
     }
 
+    #[inline]
+    pub fn to_atomic_ordering(self) -> AtomicOrdering {
+        use AtomicOrdering::*;
+        let val = self.to_u32();
+        if val == Relaxed as u32 {
+            Relaxed
+        } else if val == Release as u32 {
+            Release
+        } else if val == Acquire as u32 {
+            Acquire
+        } else if val == AcqRel as u32 {
+            AcqRel
+        } else if val == SeqCst as u32 {
+            SeqCst
+        } else {
+            panic!("not a valid atomic ordering")
+        }
+    }
+
     /// Converts the `ScalarInt` to `bool`.
     /// Panics if the `size` of the `ScalarInt` is not equal to 1 byte.
     /// Errors if it is not a valid `bool`.
@@ -422,9 +454,9 @@ macro_rules! from_scalar_int_for_x {
             impl From<ScalarInt> for $ty {
                 #[inline]
                 fn from(int: ScalarInt) -> Self {
-                    // The `unwrap` cannot fail because to_bits (if it succeeds)
+                    // The `unwrap` cannot fail because to_uint (if it succeeds)
                     // is guaranteed to return a value that fits into the size.
-                    int.to_bits(Size::from_bytes(size_of::<$ty>()))
+                    int.to_uint(Size::from_bytes(size_of::<$ty>()))
                        .try_into().unwrap()
                 }
             }
@@ -450,6 +482,49 @@ impl From<char> for ScalarInt {
     }
 }
 
+macro_rules! from_x_for_scalar_int_signed {
+    ($($ty:ty),*) => {
+        $(
+            impl From<$ty> for ScalarInt {
+                #[inline]
+                fn from(u: $ty) -> Self {
+                    Self {
+                        data: u128::from(u.cast_unsigned()), // go via the unsigned type of the same size
+                        size: NonZero::new(size_of::<$ty>() as u8).unwrap(),
+                    }
+                }
+            }
+        )*
+    }
+}
+
+macro_rules! from_scalar_int_for_x_signed {
+    ($($ty:ty),*) => {
+        $(
+            impl From<ScalarInt> for $ty {
+                #[inline]
+                fn from(int: ScalarInt) -> Self {
+                    // The `unwrap` cannot fail because to_int (if it succeeds)
+                    // is guaranteed to return a value that fits into the size.
+                    int.to_int(Size::from_bytes(size_of::<$ty>()))
+                       .try_into().unwrap()
+                }
+            }
+        )*
+    }
+}
+
+from_x_for_scalar_int_signed!(i8, i16, i32, i64, i128);
+from_scalar_int_for_x_signed!(i8, i16, i32, i64, i128);
+
+impl From<std::cmp::Ordering> for ScalarInt {
+    #[inline]
+    fn from(c: std::cmp::Ordering) -> Self {
+        // Here we rely on `cmp::Ordering` having the same values in host and target!
+        ScalarInt::from(c as i8)
+    }
+}
+
 /// Error returned when a conversion from ScalarInt to char fails.
 #[derive(Debug)]
 pub struct CharTryFromScalarInt;
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index 0759fa3da42..0b1e9852d2a 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -60,7 +60,7 @@ use tracing::{debug, instrument};
 
 use crate::arena::Arena;
 use crate::dep_graph::{DepGraph, DepKindStruct};
-use crate::infer::canonical::{CanonicalParamEnvCache, CanonicalVarInfo, CanonicalVarInfos};
+use crate::infer::canonical::{CanonicalParamEnvCache, CanonicalVarKind, CanonicalVarKinds};
 use crate::lint::lint_level;
 use crate::metadata::ModChild;
 use crate::middle::codegen_fn_attrs::{CodegenFnAttrs, TargetFeature};
@@ -107,9 +107,12 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
         self.mk_predefined_opaques_in_body(data)
     }
     type LocalDefIds = &'tcx ty::List<LocalDefId>;
-    type CanonicalVars = CanonicalVarInfos<'tcx>;
-    fn mk_canonical_var_infos(self, infos: &[ty::CanonicalVarInfo<Self>]) -> Self::CanonicalVars {
-        self.mk_canonical_var_infos(infos)
+    type CanonicalVarKinds = CanonicalVarKinds<'tcx>;
+    fn mk_canonical_var_kinds(
+        self,
+        kinds: &[ty::CanonicalVarKind<Self>],
+    ) -> Self::CanonicalVarKinds {
+        self.mk_canonical_var_kinds(kinds)
     }
 
     type ExternalConstraints = ExternalConstraints<'tcx>;
@@ -340,7 +343,7 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
     fn coroutine_hidden_types(
         self,
         def_id: DefId,
-    ) -> ty::EarlyBinder<'tcx, ty::Binder<'tcx, &'tcx ty::List<Ty<'tcx>>>> {
+    ) -> ty::EarlyBinder<'tcx, ty::Binder<'tcx, ty::CoroutineWitnessTypes<TyCtxt<'tcx>>>> {
         self.coroutine_hidden_types(def_id)
     }
 
@@ -455,7 +458,7 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
     }
 
     fn require_lang_item(self, lang_item: TraitSolverLangItem) -> DefId {
-        self.require_lang_item(trait_lang_item_to_lang_item(lang_item), None)
+        self.require_lang_item(trait_lang_item_to_lang_item(lang_item), DUMMY_SP)
     }
 
     fn is_lang_item(self, def_id: DefId, lang_item: TraitSolverLangItem) -> bool {
@@ -833,7 +836,7 @@ pub struct CtxtInterners<'tcx> {
     const_lists: InternedSet<'tcx, List<ty::Const<'tcx>>>,
     args: InternedSet<'tcx, GenericArgs<'tcx>>,
     type_lists: InternedSet<'tcx, List<Ty<'tcx>>>,
-    canonical_var_infos: InternedSet<'tcx, List<CanonicalVarInfo<'tcx>>>,
+    canonical_var_kinds: InternedSet<'tcx, List<CanonicalVarKind<'tcx>>>,
     region: InternedSet<'tcx, RegionKind<'tcx>>,
     poly_existential_predicates: InternedSet<'tcx, List<PolyExistentialPredicate<'tcx>>>,
     predicate: InternedSet<'tcx, WithCachedTypeInfo<ty::Binder<'tcx, PredicateKind<'tcx>>>>,
@@ -872,7 +875,7 @@ impl<'tcx> CtxtInterners<'tcx> {
             type_lists: InternedSet::with_capacity(N * 4),
             region: InternedSet::with_capacity(N * 4),
             poly_existential_predicates: InternedSet::with_capacity(N / 4),
-            canonical_var_infos: InternedSet::with_capacity(N / 2),
+            canonical_var_kinds: InternedSet::with_capacity(N / 2),
             predicate: InternedSet::with_capacity(N),
             clauses: InternedSet::with_capacity(N),
             projs: InternedSet::with_capacity(N * 4),
@@ -1579,7 +1582,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// Returns the same `AllocId` if called again with the same bytes.
     pub fn allocate_bytes_dedup(self, bytes: &[u8], salt: usize) -> interpret::AllocId {
         // Create an allocation that just contains these bytes.
-        let alloc = interpret::Allocation::from_bytes_byte_aligned_immutable(bytes);
+        let alloc = interpret::Allocation::from_bytes_byte_aligned_immutable(bytes, ());
         let alloc = self.mk_const_alloc(alloc);
         self.reserve_and_set_memory_dedup(alloc, salt)
     }
@@ -1707,7 +1710,7 @@ impl<'tcx> TyCtxt<'tcx> {
 
     /// Gets a `Ty` representing the [`LangItem::OrderingEnum`]
     #[track_caller]
-    pub fn ty_ordering_enum(self, span: Option<Span>) -> Ty<'tcx> {
+    pub fn ty_ordering_enum(self, span: Span) -> Ty<'tcx> {
         let ordering_enum = self.require_lang_item(hir::LangItem::OrderingEnum, span);
         self.type_of(ordering_enum).no_bound_vars().unwrap()
     }
@@ -2250,7 +2253,7 @@ impl<'tcx> TyCtxt<'tcx> {
         Ty::new_imm_ref(
             self,
             self.lifetimes.re_static,
-            self.type_of(self.require_lang_item(LangItem::PanicLocation, None))
+            self.type_of(self.require_lang_item(LangItem::PanicLocation, DUMMY_SP))
                 .instantiate(self, self.mk_args(&[self.lifetimes.re_static.into()])),
         )
     }
@@ -2675,7 +2678,7 @@ slice_interners!(
     const_lists: pub mk_const_list(Const<'tcx>),
     args: pub mk_args(GenericArg<'tcx>),
     type_lists: pub mk_type_list(Ty<'tcx>),
-    canonical_var_infos: pub mk_canonical_var_infos(CanonicalVarInfo<'tcx>),
+    canonical_var_kinds: pub mk_canonical_var_kinds(CanonicalVarKind<'tcx>),
     poly_existential_predicates: intern_poly_existential_predicates(PolyExistentialPredicate<'tcx>),
     projs: pub mk_projs(ProjectionKind),
     place_elems: pub mk_place_elems(PlaceElem<'tcx>),
@@ -2709,7 +2712,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// Given a `ty`, return whether it's an `impl Future<...>`.
     pub fn ty_is_opaque_future(self, ty: Ty<'_>) -> bool {
         let ty::Alias(ty::Opaque, ty::AliasTy { def_id, .. }) = ty.kind() else { return false };
-        let future_trait = self.require_lang_item(LangItem::Future, None);
+        let future_trait = self.require_lang_item(LangItem::Future, DUMMY_SP);
 
         self.explicit_item_self_bounds(def_id).skip_binder().iter().any(|&(predicate, _)| {
             let ty::ClauseKind::Trait(trait_predicate) = predicate.kind().skip_binder() else {
@@ -2778,7 +2781,7 @@ impl<'tcx> TyCtxt<'tcx> {
                 return false;
             }
 
-            if !matches!(args[0].unpack(), ty::GenericArgKind::Type(_)) {
+            if !matches!(args[0].kind(), ty::GenericArgKind::Type(_)) {
                 return false;
             }
 
@@ -2800,7 +2803,7 @@ impl<'tcx> TyCtxt<'tcx> {
         };
 
         for (param, arg) in std::iter::zip(&generics.own_params, own_args) {
-            match (&param.kind, arg.unpack()) {
+            match (&param.kind, arg.kind()) {
                 (ty::GenericParamDefKind::Type { .. }, ty::GenericArgKind::Type(_))
                 | (ty::GenericParamDefKind::Lifetime, ty::GenericArgKind::Lifetime(_))
                 | (ty::GenericParamDefKind::Const { .. }, ty::GenericArgKind::Const(_)) => {}
@@ -3055,9 +3058,9 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn mk_canonical_var_infos_from_iter<I, T>(self, iter: I) -> T::Output
     where
         I: Iterator<Item = T>,
-        T: CollectAndApply<CanonicalVarInfo<'tcx>, &'tcx List<CanonicalVarInfo<'tcx>>>,
+        T: CollectAndApply<CanonicalVarKind<'tcx>, &'tcx List<CanonicalVarKind<'tcx>>>,
     {
-        T::collect_and_apply(iter, |xs| self.mk_canonical_var_infos(xs))
+        T::collect_and_apply(iter, |xs| self.mk_canonical_var_kinds(xs))
     }
 
     pub fn mk_place_elems_from_iter<I, T>(self, iter: I) -> T::Output
diff --git a/compiler/rustc_middle/src/ty/erase_regions.rs b/compiler/rustc_middle/src/ty/erase_regions.rs
index 45a0b1288db..f4fead7e952 100644
--- a/compiler/rustc_middle/src/ty/erase_regions.rs
+++ b/compiler/rustc_middle/src/ty/erase_regions.rs
@@ -86,4 +86,12 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for RegionEraserVisitor<'tcx> {
             p
         }
     }
+
+    fn fold_clauses(&mut self, c: ty::Clauses<'tcx>) -> ty::Clauses<'tcx> {
+        if c.has_type_flags(TypeFlags::HAS_BINDER_VARS | TypeFlags::HAS_FREE_REGIONS) {
+            c.super_fold_with(self)
+        } else {
+            c
+        }
+    }
 }
diff --git a/compiler/rustc_middle/src/ty/fold.rs b/compiler/rustc_middle/src/ty/fold.rs
index 8d6871d2f1f..b2057fa36d7 100644
--- a/compiler/rustc_middle/src/ty/fold.rs
+++ b/compiler/rustc_middle/src/ty/fold.rs
@@ -177,6 +177,10 @@ where
     fn fold_predicate(&mut self, p: ty::Predicate<'tcx>) -> ty::Predicate<'tcx> {
         if p.has_vars_bound_at_or_above(self.current_index) { p.super_fold_with(self) } else { p }
     }
+
+    fn fold_clauses(&mut self, c: ty::Clauses<'tcx>) -> ty::Clauses<'tcx> {
+        if c.has_vars_bound_at_or_above(self.current_index) { c.super_fold_with(self) } else { c }
+    }
 }
 
 impl<'tcx> TyCtxt<'tcx> {
diff --git a/compiler/rustc_middle/src/ty/generic_args.rs b/compiler/rustc_middle/src/ty/generic_args.rs
index 542c0b3e6eb..5e038f91675 100644
--- a/compiler/rustc_middle/src/ty/generic_args.rs
+++ b/compiler/rustc_middle/src/ty/generic_args.rs
@@ -137,7 +137,7 @@ impl<'tcx> rustc_type_ir::inherent::IntoKind for GenericArg<'tcx> {
     type Kind = GenericArgKind<'tcx>;
 
     fn kind(self) -> Self::Kind {
-        self.unpack()
+        self.kind()
     }
 }
 
@@ -218,7 +218,7 @@ impl<'tcx> From<ty::Const<'tcx>> for GenericArg<'tcx> {
 
 impl<'tcx> From<ty::Term<'tcx>> for GenericArg<'tcx> {
     fn from(value: ty::Term<'tcx>) -> Self {
-        match value.unpack() {
+        match value.kind() {
             ty::TermKind::Ty(t) => t.into(),
             ty::TermKind::Const(c) => c.into(),
         }
@@ -227,7 +227,7 @@ impl<'tcx> From<ty::Term<'tcx>> for GenericArg<'tcx> {
 
 impl<'tcx> GenericArg<'tcx> {
     #[inline]
-    pub fn unpack(self) -> GenericArgKind<'tcx> {
+    pub fn kind(self) -> GenericArgKind<'tcx> {
         let ptr =
             unsafe { self.ptr.map_addr(|addr| NonZero::new_unchecked(addr.get() & !TAG_MASK)) };
         // SAFETY: use of `Interned::new_unchecked` here is ok because these
@@ -251,7 +251,7 @@ impl<'tcx> GenericArg<'tcx> {
 
     #[inline]
     pub fn as_region(self) -> Option<ty::Region<'tcx>> {
-        match self.unpack() {
+        match self.kind() {
             GenericArgKind::Lifetime(re) => Some(re),
             _ => None,
         }
@@ -259,7 +259,7 @@ impl<'tcx> GenericArg<'tcx> {
 
     #[inline]
     pub fn as_type(self) -> Option<Ty<'tcx>> {
-        match self.unpack() {
+        match self.kind() {
             GenericArgKind::Type(ty) => Some(ty),
             _ => None,
         }
@@ -267,7 +267,7 @@ impl<'tcx> GenericArg<'tcx> {
 
     #[inline]
     pub fn as_const(self) -> Option<ty::Const<'tcx>> {
-        match self.unpack() {
+        match self.kind() {
             GenericArgKind::Const(ct) => Some(ct),
             _ => None,
         }
@@ -275,7 +275,7 @@ impl<'tcx> GenericArg<'tcx> {
 
     #[inline]
     pub fn as_term(self) -> Option<ty::Term<'tcx>> {
-        match self.unpack() {
+        match self.kind() {
             GenericArgKind::Lifetime(_) => None,
             GenericArgKind::Type(ty) => Some(ty.into()),
             GenericArgKind::Const(ct) => Some(ct.into()),
@@ -300,7 +300,7 @@ impl<'tcx> GenericArg<'tcx> {
     }
 
     pub fn is_non_region_infer(self) -> bool {
-        match self.unpack() {
+        match self.kind() {
             GenericArgKind::Lifetime(_) => false,
             // FIXME: This shouldn't return numerical/float.
             GenericArgKind::Type(ty) => ty.is_ty_or_numeric_infer(),
@@ -327,7 +327,7 @@ impl<'a, 'tcx> Lift<TyCtxt<'tcx>> for GenericArg<'a> {
     type Lifted = GenericArg<'tcx>;
 
     fn lift_to_interner(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        match self.unpack() {
+        match self.kind() {
             GenericArgKind::Lifetime(lt) => tcx.lift(lt).map(|lt| lt.into()),
             GenericArgKind::Type(ty) => tcx.lift(ty).map(|ty| ty.into()),
             GenericArgKind::Const(ct) => tcx.lift(ct).map(|ct| ct.into()),
@@ -340,7 +340,7 @@ impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for GenericArg<'tcx> {
         self,
         folder: &mut F,
     ) -> Result<Self, F::Error> {
-        match self.unpack() {
+        match self.kind() {
             GenericArgKind::Lifetime(lt) => lt.try_fold_with(folder).map(Into::into),
             GenericArgKind::Type(ty) => ty.try_fold_with(folder).map(Into::into),
             GenericArgKind::Const(ct) => ct.try_fold_with(folder).map(Into::into),
@@ -348,7 +348,7 @@ impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for GenericArg<'tcx> {
     }
 
     fn fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, folder: &mut F) -> Self {
-        match self.unpack() {
+        match self.kind() {
             GenericArgKind::Lifetime(lt) => lt.fold_with(folder).into(),
             GenericArgKind::Type(ty) => ty.fold_with(folder).into(),
             GenericArgKind::Const(ct) => ct.fold_with(folder).into(),
@@ -358,7 +358,7 @@ impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for GenericArg<'tcx> {
 
 impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for GenericArg<'tcx> {
     fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(&self, visitor: &mut V) -> V::Result {
-        match self.unpack() {
+        match self.kind() {
             GenericArgKind::Lifetime(lt) => lt.visit_with(visitor),
             GenericArgKind::Type(ty) => ty.visit_with(visitor),
             GenericArgKind::Const(ct) => ct.visit_with(visitor),
@@ -368,7 +368,7 @@ impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for GenericArg<'tcx> {
 
 impl<'tcx, E: TyEncoder<'tcx>> Encodable<E> for GenericArg<'tcx> {
     fn encode(&self, e: &mut E) {
-        self.unpack().encode(e)
+        self.kind().encode(e)
     }
 }
 
@@ -390,7 +390,7 @@ impl<'tcx> GenericArgs<'tcx> {
     ///
     /// If any of the generic arguments are not types.
     pub fn into_type_list(&self, tcx: TyCtxt<'tcx>) -> &'tcx List<Ty<'tcx>> {
-        tcx.mk_type_list_from_iter(self.iter().map(|arg| match arg.unpack() {
+        tcx.mk_type_list_from_iter(self.iter().map(|arg| match arg.kind() {
             GenericArgKind::Type(ty) => ty,
             _ => bug!("`into_type_list` called on generic arg with non-types"),
         }))
@@ -527,7 +527,7 @@ impl<'tcx> GenericArgs<'tcx> {
     /// Returns generic arguments that are not lifetimes.
     #[inline]
     pub fn non_erasable_generics(&self) -> impl DoubleEndedIterator<Item = GenericArgKind<'tcx>> {
-        self.iter().filter_map(|k| match k.unpack() {
+        self.iter().filter_map(|arg| match arg.kind() {
             ty::GenericArgKind::Lifetime(_) => None,
             generic => Some(generic),
         })
diff --git a/compiler/rustc_middle/src/ty/impls_ty.rs b/compiler/rustc_middle/src/ty/impls_ty.rs
index 5f6305bb48a..ac45ce887c9 100644
--- a/compiler/rustc_middle/src/ty/impls_ty.rs
+++ b/compiler/rustc_middle/src/ty/impls_ty.rs
@@ -60,7 +60,7 @@ where
 
 impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for ty::GenericArg<'tcx> {
     fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
-        self.unpack().hash_stable(hcx, hasher);
+        self.kind().hash_stable(hcx, hasher);
     }
 }
 
diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs
index 0d99a1b5149..5ba4e5446e9 100644
--- a/compiler/rustc_middle/src/ty/instance.rs
+++ b/compiler/rustc_middle/src/ty/instance.rs
@@ -786,7 +786,7 @@ impl<'tcx> Instance<'tcx> {
     }
 
     pub fn resolve_drop_in_place(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ty::Instance<'tcx> {
-        let def_id = tcx.require_lang_item(LangItem::DropInPlace, None);
+        let def_id = tcx.require_lang_item(LangItem::DropInPlace, DUMMY_SP);
         let args = tcx.mk_args(&[ty.into()]);
         Instance::expect_resolve(
             tcx,
@@ -798,7 +798,7 @@ impl<'tcx> Instance<'tcx> {
     }
 
     pub fn resolve_async_drop_in_place(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ty::Instance<'tcx> {
-        let def_id = tcx.require_lang_item(LangItem::AsyncDropInPlace, None);
+        let def_id = tcx.require_lang_item(LangItem::AsyncDropInPlace, DUMMY_SP);
         let args = tcx.mk_args(&[ty.into()]);
         Instance::expect_resolve(
             tcx,
@@ -824,7 +824,7 @@ impl<'tcx> Instance<'tcx> {
         closure_did: DefId,
         args: ty::GenericArgsRef<'tcx>,
     ) -> Instance<'tcx> {
-        let fn_once = tcx.require_lang_item(LangItem::FnOnce, None);
+        let fn_once = tcx.require_lang_item(LangItem::FnOnce, DUMMY_SP);
         let call_once = tcx
             .associated_items(fn_once)
             .in_definition_order()
diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs
index 7ebfebea44e..c2ae6b06192 100644
--- a/compiler/rustc_middle/src/ty/layout.rs
+++ b/compiler/rustc_middle/src/ty/layout.rs
@@ -934,7 +934,7 @@ where
                             .unwrap(),
                     ),
                     Variants::Multiple { tag, tag_field, .. } => {
-                        if i == tag_field {
+                        if FieldIdx::from_usize(i) == tag_field {
                             return TyMaybeWithLayout::TyAndLayout(tag_layout(tag));
                         }
                         TyMaybeWithLayout::Ty(args.as_coroutine().prefix_tys()[i])
@@ -1060,8 +1060,10 @@ where
                         tag_field,
                         variants,
                         ..
-                    } if variants.len() == 2 && this.fields.offset(*tag_field) == offset => {
-                        let tagged_variant = if untagged_variant.as_u32() == 0 {
+                    } if variants.len() == 2
+                        && this.fields.offset(tag_field.as_usize()) == offset =>
+                    {
+                        let tagged_variant = if *untagged_variant == VariantIdx::ZERO {
                             VariantIdx::from_u32(1)
                         } else {
                             VariantIdx::from_u32(0)
diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs
index b2a58897c31..af31f7ed33b 100644
--- a/compiler/rustc_middle/src/ty/mod.rs
+++ b/compiler/rustc_middle/src/ty/mod.rs
@@ -74,8 +74,8 @@ pub use self::closure::{
     place_to_string_for_capture,
 };
 pub use self::consts::{
-    Const, ConstInt, ConstKind, Expr, ExprKind, ScalarInt, UnevaluatedConst, ValTree, ValTreeKind,
-    Value,
+    AnonConstKind, AtomicOrdering, Const, ConstInt, ConstKind, Expr, ExprKind, ScalarInt,
+    UnevaluatedConst, ValTree, ValTreeKind, Value,
 };
 pub use self::context::{
     CtxtInterners, CurrentGcx, DeducedParamAttrs, Feed, FreeRegionInfo, GlobalCtxt, Lift, TyCtxt,
@@ -504,7 +504,7 @@ impl<'tcx> rustc_type_ir::inherent::IntoKind for Term<'tcx> {
     type Kind = TermKind<'tcx>;
 
     fn kind(self) -> Self::Kind {
-        self.unpack()
+        self.kind()
     }
 }
 
@@ -521,7 +521,7 @@ unsafe impl<'tcx> Sync for Term<'tcx> where &'tcx (Ty<'tcx>, Const<'tcx>): Sync
 
 impl Debug for Term<'_> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self.unpack() {
+        match self.kind() {
             TermKind::Ty(ty) => write!(f, "Term::Ty({ty:?})"),
             TermKind::Const(ct) => write!(f, "Term::Const({ct:?})"),
         }
@@ -542,7 +542,7 @@ impl<'tcx> From<Const<'tcx>> for Term<'tcx> {
 
 impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for Term<'tcx> {
     fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
-        self.unpack().hash_stable(hcx, hasher);
+        self.kind().hash_stable(hcx, hasher);
     }
 }
 
@@ -551,14 +551,14 @@ impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for Term<'tcx> {
         self,
         folder: &mut F,
     ) -> Result<Self, F::Error> {
-        match self.unpack() {
+        match self.kind() {
             ty::TermKind::Ty(ty) => ty.try_fold_with(folder).map(Into::into),
             ty::TermKind::Const(ct) => ct.try_fold_with(folder).map(Into::into),
         }
     }
 
     fn fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, folder: &mut F) -> Self {
-        match self.unpack() {
+        match self.kind() {
             ty::TermKind::Ty(ty) => ty.fold_with(folder).into(),
             ty::TermKind::Const(ct) => ct.fold_with(folder).into(),
         }
@@ -567,7 +567,7 @@ impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for Term<'tcx> {
 
 impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for Term<'tcx> {
     fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(&self, visitor: &mut V) -> V::Result {
-        match self.unpack() {
+        match self.kind() {
             ty::TermKind::Ty(ty) => ty.visit_with(visitor),
             ty::TermKind::Const(ct) => ct.visit_with(visitor),
         }
@@ -576,7 +576,7 @@ impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for Term<'tcx> {
 
 impl<'tcx, E: TyEncoder<'tcx>> Encodable<E> for Term<'tcx> {
     fn encode(&self, e: &mut E) {
-        self.unpack().encode(e)
+        self.kind().encode(e)
     }
 }
 
@@ -589,7 +589,7 @@ impl<'tcx, D: TyDecoder<'tcx>> Decodable<D> for Term<'tcx> {
 
 impl<'tcx> Term<'tcx> {
     #[inline]
-    pub fn unpack(self) -> TermKind<'tcx> {
+    pub fn kind(self) -> TermKind<'tcx> {
         let ptr =
             unsafe { self.ptr.map_addr(|addr| NonZero::new_unchecked(addr.get() & !TAG_MASK)) };
         // SAFETY: use of `Interned::new_unchecked` here is ok because these
@@ -609,7 +609,7 @@ impl<'tcx> Term<'tcx> {
     }
 
     pub fn as_type(&self) -> Option<Ty<'tcx>> {
-        if let TermKind::Ty(ty) = self.unpack() { Some(ty) } else { None }
+        if let TermKind::Ty(ty) = self.kind() { Some(ty) } else { None }
     }
 
     pub fn expect_type(&self) -> Ty<'tcx> {
@@ -617,7 +617,7 @@ impl<'tcx> Term<'tcx> {
     }
 
     pub fn as_const(&self) -> Option<Const<'tcx>> {
-        if let TermKind::Const(c) = self.unpack() { Some(c) } else { None }
+        if let TermKind::Const(c) = self.kind() { Some(c) } else { None }
     }
 
     pub fn expect_const(&self) -> Const<'tcx> {
@@ -625,14 +625,14 @@ impl<'tcx> Term<'tcx> {
     }
 
     pub fn into_arg(self) -> GenericArg<'tcx> {
-        match self.unpack() {
+        match self.kind() {
             TermKind::Ty(ty) => ty.into(),
             TermKind::Const(c) => c.into(),
         }
     }
 
     pub fn to_alias_term(self) -> Option<AliasTerm<'tcx>> {
-        match self.unpack() {
+        match self.kind() {
             TermKind::Ty(ty) => match *ty.kind() {
                 ty::Alias(_kind, alias_ty) => Some(alias_ty.into()),
                 _ => None,
@@ -645,7 +645,7 @@ impl<'tcx> Term<'tcx> {
     }
 
     pub fn is_infer(&self) -> bool {
-        match self.unpack() {
+        match self.kind() {
             TermKind::Ty(ty) => ty.is_ty_var(),
             TermKind::Const(ct) => ct.is_ct_infer(),
         }
diff --git a/compiler/rustc_middle/src/ty/opaque_types.rs b/compiler/rustc_middle/src/ty/opaque_types.rs
index 9445a18ad76..2b024b7b6cb 100644
--- a/compiler/rustc_middle/src/ty/opaque_types.rs
+++ b/compiler/rustc_middle/src/ty/opaque_types.rs
@@ -120,7 +120,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ReverseMapper<'tcx> {
             }
         }
 
-        match self.map.get(&r.into()).map(|k| k.unpack()) {
+        match self.map.get(&r.into()).map(|arg| arg.kind()) {
             Some(GenericArgKind::Lifetime(r1)) => r1,
             Some(u) => panic!("region mapped to unexpected kind: {u:?}"),
             None if self.do_not_error => self.tcx.lifetimes.re_static,
@@ -162,7 +162,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ReverseMapper<'tcx> {
 
             ty::Param(param) => {
                 // Look it up in the generic parameters list.
-                match self.map.get(&ty.into()).map(|k| k.unpack()) {
+                match self.map.get(&ty.into()).map(|arg| arg.kind()) {
                     // Found it in the generic parameters list; replace with the parameter from the
                     // opaque type.
                     Some(GenericArgKind::Type(t1)) => t1,
@@ -195,7 +195,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ReverseMapper<'tcx> {
         match ct.kind() {
             ty::ConstKind::Param(..) => {
                 // Look it up in the generic parameters list.
-                match self.map.get(&ct.into()).map(|k| k.unpack()) {
+                match self.map.get(&ct.into()).map(|arg| arg.kind()) {
                     // Found it in the generic parameters list, replace with the parameter from the
                     // opaque type.
                     Some(GenericArgKind::Const(c1)) => c1,
diff --git a/compiler/rustc_middle/src/ty/parameterized.rs b/compiler/rustc_middle/src/ty/parameterized.rs
index ecd6132b3ef..3858778bfc8 100644
--- a/compiler/rustc_middle/src/ty/parameterized.rs
+++ b/compiler/rustc_middle/src/ty/parameterized.rs
@@ -68,6 +68,7 @@ trivially_parameterized_over_tcx! {
     ty::AsyncDestructor,
     ty::AssocItemContainer,
     ty::Asyncness,
+    ty::AnonConstKind,
     ty::DeducedParamAttrs,
     ty::Destructor,
     ty::Generics,
diff --git a/compiler/rustc_middle/src/ty/predicate.rs b/compiler/rustc_middle/src/ty/predicate.rs
index 551d816941b..bc2ac42b6b1 100644
--- a/compiler/rustc_middle/src/ty/predicate.rs
+++ b/compiler/rustc_middle/src/ty/predicate.rs
@@ -238,6 +238,8 @@ impl<'tcx> Clause<'tcx> {
     }
 }
 
+impl<'tcx> rustc_type_ir::inherent::Clauses<TyCtxt<'tcx>> for ty::Clauses<'tcx> {}
+
 #[extension(pub trait ExistentialPredicateStableCmpExt<'tcx>)]
 impl<'tcx> ExistentialPredicate<'tcx> {
     /// Compares via an ordering that will not change if modules are reordered or other changes are
diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs
index 6fd6aff0e2b..673a89a8134 100644
--- a/compiler/rustc_middle/src/ty/print/pretty.rs
+++ b/compiler/rustc_middle/src/ty/print/pretty.rs
@@ -1239,7 +1239,7 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
 
                         p!(write("{} = ", tcx.associated_item(assoc_item_def_id).name()));
 
-                        match term.unpack() {
+                        match term.kind() {
                             TermKind::Ty(ty) => p!(print(ty)),
                             TermKind::Const(c) => p!(print(c)),
                         };
@@ -1886,7 +1886,7 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
     ) -> Result<(), PrintError> {
         define_scoped_cx!(self);
 
-        if self.should_print_verbose() {
+        if with_reduced_queries() || self.should_print_verbose() {
             p!(write("ValTree({:?}: ", cv.valtree), print(cv.ty), ")");
             return Ok(());
         }
@@ -3386,7 +3386,7 @@ define_print_and_forward_display! {
     }
 
     ty::Term<'tcx> {
-      match self.unpack() {
+      match self.kind() {
         ty::TermKind::Ty(ty) => p!(print(ty)),
         ty::TermKind::Const(c) => p!(print(c)),
       }
@@ -3401,7 +3401,7 @@ define_print_and_forward_display! {
     }
 
     GenericArg<'tcx> {
-        match self.unpack() {
+        match self.kind() {
             GenericArgKind::Lifetime(lt) => p!(print(lt)),
             GenericArgKind::Type(ty) => p!(print(ty)),
             GenericArgKind::Const(ct) => p!(print(ct)),
diff --git a/compiler/rustc_middle/src/ty/relate.rs b/compiler/rustc_middle/src/ty/relate.rs
index 6ad4e5276b2..dc1d60f3d43 100644
--- a/compiler/rustc_middle/src/ty/relate.rs
+++ b/compiler/rustc_middle/src/ty/relate.rs
@@ -169,7 +169,7 @@ impl<'tcx> Relate<TyCtxt<'tcx>> for ty::GenericArg<'tcx> {
         a: ty::GenericArg<'tcx>,
         b: ty::GenericArg<'tcx>,
     ) -> RelateResult<'tcx, ty::GenericArg<'tcx>> {
-        match (a.unpack(), b.unpack()) {
+        match (a.kind(), b.kind()) {
             (ty::GenericArgKind::Lifetime(a_lt), ty::GenericArgKind::Lifetime(b_lt)) => {
                 Ok(relation.relate(a_lt, b_lt)?.into())
             }
@@ -190,7 +190,7 @@ impl<'tcx> Relate<TyCtxt<'tcx>> for ty::Term<'tcx> {
         a: Self,
         b: Self,
     ) -> RelateResult<'tcx, Self> {
-        Ok(match (a.unpack(), b.unpack()) {
+        Ok(match (a.kind(), b.kind()) {
             (ty::TermKind::Ty(a), ty::TermKind::Ty(b)) => relation.relate(a, b)?.into(),
             (ty::TermKind::Const(a), ty::TermKind::Const(b)) => relation.relate(a, b)?.into(),
             _ => return Err(TypeError::Mismatch),
diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs
index 58f7bc75054..000ba7b6fa7 100644
--- a/compiler/rustc_middle/src/ty/structural_impls.rs
+++ b/compiler/rustc_middle/src/ty/structural_impls.rs
@@ -202,7 +202,7 @@ impl<T: fmt::Debug> fmt::Debug for ty::Placeholder<T> {
 
 impl<'tcx> fmt::Debug for GenericArg<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self.unpack() {
+        match self.kind() {
             GenericArgKind::Lifetime(lt) => lt.fmt(f),
             GenericArgKind::Type(ty) => ty.fmt(f),
             GenericArgKind::Const(ct) => ct.fmt(f),
@@ -326,7 +326,7 @@ impl<'tcx, T: Lift<TyCtxt<'tcx>>> Lift<TyCtxt<'tcx>> for Option<T> {
 impl<'a, 'tcx> Lift<TyCtxt<'tcx>> for Term<'a> {
     type Lifted = ty::Term<'tcx>;
     fn lift_to_interner(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        match self.unpack() {
+        match self.kind() {
             TermKind::Ty(ty) => tcx.lift(ty).map(Into::into),
             TermKind::Const(c) => tcx.lift(c).map(Into::into),
         }
@@ -570,6 +570,19 @@ impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for ty::Clause<'tcx> {
     }
 }
 
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for ty::Clauses<'tcx> {
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+        self,
+        folder: &mut F,
+    ) -> Result<Self, F::Error> {
+        folder.try_fold_clauses(self)
+    }
+
+    fn fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, folder: &mut F) -> Self {
+        folder.fold_clauses(self)
+    }
+}
+
 impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for ty::Predicate<'tcx> {
     fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(&self, visitor: &mut V) -> V::Result {
         visitor.visit_predicate(*self)
@@ -615,6 +628,19 @@ impl<'tcx> TypeSuperVisitable<TyCtxt<'tcx>> for ty::Clauses<'tcx> {
     }
 }
 
+impl<'tcx> TypeSuperFoldable<TyCtxt<'tcx>> for ty::Clauses<'tcx> {
+    fn try_super_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+        self,
+        folder: &mut F,
+    ) -> Result<Self, F::Error> {
+        ty::util::try_fold_list(self, folder, |tcx, v| tcx.mk_clauses(v))
+    }
+
+    fn super_fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, folder: &mut F) -> Self {
+        ty::util::fold_list(self, folder, |tcx, v| tcx.mk_clauses(v))
+    }
+}
+
 impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for ty::Const<'tcx> {
     fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
         self,
@@ -775,7 +801,6 @@ macro_rules! list_fold {
 }
 
 list_fold! {
-    ty::Clauses<'tcx> : mk_clauses,
     &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>> : mk_poly_existential_predicates,
     &'tcx ty::List<PlaceElem<'tcx>> : mk_place_elems,
     &'tcx ty::List<ty::Pattern<'tcx>> : mk_patterns,
diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs
index ab1f3d6099f..cbf545c01c5 100644
--- a/compiler/rustc_middle/src/ty/sty.rs
+++ b/compiler/rustc_middle/src/ty/sty.rs
@@ -593,7 +593,7 @@ impl<'tcx> Ty<'tcx> {
         ty: Ty<'tcx>,
         mutbl: ty::Mutability,
     ) -> Ty<'tcx> {
-        let pin = tcx.adt_def(tcx.require_lang_item(LangItem::Pin, None));
+        let pin = tcx.adt_def(tcx.require_lang_item(LangItem::Pin, DUMMY_SP));
         Ty::new_adt(tcx, pin, tcx.mk_args(&[Ty::new_ref(tcx, r, ty, mutbl).into()]))
     }
 
@@ -857,19 +857,19 @@ impl<'tcx> Ty<'tcx> {
 
     #[inline]
     pub fn new_box(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Ty<'tcx> {
-        let def_id = tcx.require_lang_item(LangItem::OwnedBox, None);
+        let def_id = tcx.require_lang_item(LangItem::OwnedBox, DUMMY_SP);
         Ty::new_generic_adt(tcx, def_id, ty)
     }
 
     #[inline]
     pub fn new_maybe_uninit(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Ty<'tcx> {
-        let def_id = tcx.require_lang_item(LangItem::MaybeUninit, None);
+        let def_id = tcx.require_lang_item(LangItem::MaybeUninit, DUMMY_SP);
         Ty::new_generic_adt(tcx, def_id, ty)
     }
 
     /// Creates a `&mut Context<'_>` [`Ty`] with erased lifetimes.
     pub fn new_task_context(tcx: TyCtxt<'tcx>) -> Ty<'tcx> {
-        let context_did = tcx.require_lang_item(LangItem::Context, None);
+        let context_did = tcx.require_lang_item(LangItem::Context, DUMMY_SP);
         let context_adt_ref = tcx.adt_def(context_did);
         let context_args = tcx.mk_args(&[tcx.lifetimes.re_erased.into()]);
         let context_ty = Ty::new_adt(tcx, context_adt_ref, context_args);
@@ -1549,7 +1549,7 @@ impl<'tcx> Ty<'tcx> {
 
             ty::Param(_) | ty::Alias(..) | ty::Infer(ty::TyVar(_)) => {
                 let assoc_items = tcx.associated_item_def_ids(
-                    tcx.require_lang_item(hir::LangItem::DiscriminantKind, None),
+                    tcx.require_lang_item(hir::LangItem::DiscriminantKind, DUMMY_SP),
                 );
                 Ty::new_projection_from_args(tcx, assoc_items[0], tcx.mk_args(&[self.into()]))
             }
@@ -1629,7 +1629,7 @@ impl<'tcx> Ty<'tcx> {
             ty::Str | ty::Slice(_) => Ok(tcx.types.usize),
 
             ty::Dynamic(_, _, ty::Dyn) => {
-                let dyn_metadata = tcx.require_lang_item(LangItem::DynMetadata, None);
+                let dyn_metadata = tcx.require_lang_item(LangItem::DynMetadata, DUMMY_SP);
                 Ok(tcx.type_of(dyn_metadata).instantiate(tcx, &[tail.into()]))
             }
 
@@ -1683,7 +1683,7 @@ impl<'tcx> Ty<'tcx> {
             match pointee_ty.ptr_metadata_ty_or_tail(tcx, |x| x) {
                 Ok(metadata_ty) => metadata_ty,
                 Err(tail_ty) => {
-                    let metadata_def_id = tcx.require_lang_item(LangItem::Metadata, None);
+                    let metadata_def_id = tcx.require_lang_item(LangItem::Metadata, DUMMY_SP);
                     Ty::new_projection(tcx, metadata_def_id, [tail_ty])
                 }
             }
@@ -1882,10 +1882,8 @@ impl<'tcx> Ty<'tcx> {
             // Needs normalization or revealing to determine, so no is the safe answer.
             ty::Alias(..) => false,
 
-            ty::Param(..) | ty::Infer(..) | ty::Error(..) => false,
-
-            ty::Bound(..) | ty::Placeholder(..) => {
-                bug!("`is_trivially_pure_clone_copy` applied to unexpected type: {:?}", self);
+            ty::Param(..) | ty::Placeholder(..) | ty::Bound(..) | ty::Infer(..) | ty::Error(..) => {
+                false
             }
         }
     }
diff --git a/compiler/rustc_middle/src/ty/typeck_results.rs b/compiler/rustc_middle/src/ty/typeck_results.rs
index c6a45f84686..88583407d25 100644
--- a/compiler/rustc_middle/src/ty/typeck_results.rs
+++ b/compiler/rustc_middle/src/ty/typeck_results.rs
@@ -199,7 +199,7 @@ pub struct TypeckResults<'tcx> {
 
     /// Tracks the rvalue scoping rules which defines finer scoping for rvalue expressions
     /// by applying extended parameter rules.
-    /// Details may be find in `rustc_hir_analysis::check::rvalue_scopes`.
+    /// Details may be found in `rustc_hir_analysis::check::rvalue_scopes`.
     pub rvalue_scopes: RvalueScopes,
 
     /// Stores the predicates that apply on coroutine witness types.
@@ -777,8 +777,8 @@ impl<'tcx> IsIdentity for CanonicalUserType<'tcx> {
                     return false;
                 }
 
-                iter::zip(user_args.args, BoundVar::ZERO..).all(|(kind, cvar)| {
-                    match kind.unpack() {
+                iter::zip(user_args.args, BoundVar::ZERO..).all(|(arg, cvar)| {
+                    match arg.kind() {
                         GenericArgKind::Type(ty) => match ty.kind() {
                             ty::Bound(debruijn, b) => {
                                 // We only allow a `ty::INNERMOST` index in generic parameters.
diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index 9676aa40448..461d92f8006 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -26,7 +26,7 @@ use crate::query::Providers;
 use crate::ty::layout::{FloatExt, IntegerExt};
 use crate::ty::{
     self, Asyncness, FallibleTypeFolder, GenericArgKind, GenericArgsRef, Ty, TyCtxt, TypeFoldable,
-    TypeFolder, TypeSuperFoldable, TypeVisitableExt, Upcast, fold_regions,
+    TypeFolder, TypeSuperFoldable, TypeVisitableExt, Upcast,
 };
 
 #[derive(Copy, Clone, Debug)]
@@ -516,8 +516,8 @@ impl<'tcx> TyCtxt<'tcx> {
         let item_args = ty::GenericArgs::identity_for_item(self, def.did());
 
         let result = iter::zip(item_args, impl_args)
-            .filter(|&(_, k)| {
-                match k.unpack() {
+            .filter(|&(_, arg)| {
+                match arg.kind() {
                     GenericArgKind::Lifetime(region) => match region.kind() {
                         ty::ReEarlyParam(ebr) => {
                             !impl_generics.region_param(ebr, self).pure_wrt_drop
@@ -554,7 +554,7 @@ impl<'tcx> TyCtxt<'tcx> {
         let mut seen = GrowableBitSet::default();
         let mut seen_late = FxHashSet::default();
         for arg in args {
-            match arg.unpack() {
+            match arg.kind() {
                 GenericArgKind::Lifetime(lt) => match (ignore_regions, lt.kind()) {
                     (CheckRegions::FromFunction, ty::ReBound(di, reg)) => {
                         if !seen_late.insert((di, reg)) {
@@ -737,40 +737,6 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
-    /// Return the set of types that should be taken into account when checking
-    /// trait bounds on a coroutine's internal state. This properly replaces
-    /// `ReErased` with new existential bound lifetimes.
-    pub fn coroutine_hidden_types(
-        self,
-        def_id: DefId,
-    ) -> ty::EarlyBinder<'tcx, ty::Binder<'tcx, &'tcx ty::List<Ty<'tcx>>>> {
-        let coroutine_layout = self.mir_coroutine_witnesses(def_id);
-        let mut vars = vec![];
-        let bound_tys = self.mk_type_list_from_iter(
-            coroutine_layout
-                .as_ref()
-                .map_or_else(|| [].iter(), |l| l.field_tys.iter())
-                .filter(|decl| !decl.ignore_for_traits)
-                .map(|decl| {
-                    let ty = fold_regions(self, decl.ty, |re, debruijn| {
-                        assert_eq!(re, self.lifetimes.re_erased);
-                        let var = ty::BoundVar::from_usize(vars.len());
-                        vars.push(ty::BoundVariableKind::Region(ty::BoundRegionKind::Anon));
-                        ty::Region::new_bound(
-                            self,
-                            debruijn,
-                            ty::BoundRegion { var, kind: ty::BoundRegionKind::Anon },
-                        )
-                    });
-                    ty
-                }),
-        );
-        ty::EarlyBinder::bind(ty::Binder::bind_with_vars(
-            bound_tys,
-            self.mk_bound_variable_kinds(&vars),
-        ))
-    }
-
     /// Expands the given impl trait type, stopping if the type is recursive.
     #[instrument(skip(self), level = "debug", ret)]
     pub fn try_expand_impl_trait_type(
diff --git a/compiler/rustc_middle/src/ty/vtable.rs b/compiler/rustc_middle/src/ty/vtable.rs
index 6c9e0e7c0eb..74b6a840a2e 100644
--- a/compiler/rustc_middle/src/ty/vtable.rs
+++ b/compiler/rustc_middle/src/ty/vtable.rs
@@ -110,7 +110,7 @@ pub(super) fn vtable_allocation_provider<'tcx>(
     let ptr_align = tcx.data_layout.pointer_align.abi;
 
     let vtable_size = ptr_size * u64::try_from(vtable_entries.len()).unwrap();
-    let mut vtable = Allocation::new(vtable_size, ptr_align, AllocInit::Uninit);
+    let mut vtable = Allocation::new(vtable_size, ptr_align, AllocInit::Uninit, ());
 
     // No need to do any alignment checks on the memory accesses below, because we know the
     // allocation is correctly aligned as we created it above. Also we're only offsetting by
diff --git a/compiler/rustc_mir_build/src/builder/custom/parse/instruction.rs b/compiler/rustc_mir_build/src/builder/custom/parse/instruction.rs
index 494ee33fd8b..9825b947fe0 100644
--- a/compiler/rustc_mir_build/src/builder/custom/parse/instruction.rs
+++ b/compiler/rustc_mir_build/src/builder/custom/parse/instruction.rs
@@ -323,9 +323,9 @@ impl<'a, 'tcx> ParseCtxt<'a, 'tcx> {
     fn parse_place_inner(&self, expr_id: ExprId) -> PResult<(Place<'tcx>, PlaceTy<'tcx>)> {
         let (parent, proj) = parse_by_kind!(self, expr_id, expr, "place",
             @call(mir_field, args) => {
-                let (parent, ty) = self.parse_place_inner(args[0])?;
+                let (parent, place_ty) = self.parse_place_inner(args[0])?;
                 let field = FieldIdx::from_u32(self.parse_integer_literal(args[1])? as u32);
-                let field_ty = ty.field_ty(self.tcx, field);
+                let field_ty = PlaceTy::field_ty(self.tcx, place_ty.ty, place_ty.variant_index, field);
                 let proj = PlaceElem::Field(field, field_ty);
                 let place = parent.project_deeper(&[proj], self.tcx);
                 return Ok((place, PlaceTy::from_ty(field_ty)));
diff --git a/compiler/rustc_mir_build/src/builder/expr/as_constant.rs b/compiler/rustc_mir_build/src/builder/expr/as_constant.rs
index 64d092e0354..eb8e98ec364 100644
--- a/compiler/rustc_mir_build/src/builder/expr/as_constant.rs
+++ b/compiler/rustc_mir_build/src/builder/expr/as_constant.rs
@@ -121,14 +121,14 @@ fn lit_to_mir_constant<'tcx>(tcx: TyCtxt<'tcx>, lit_input: LitToConstInput<'tcx>
     let value = match (lit, lit_ty.kind()) {
         (ast::LitKind::Str(s, _), ty::Ref(_, inner_ty, _)) if inner_ty.is_str() => {
             let s = s.as_str();
-            let allocation = Allocation::from_bytes_byte_aligned_immutable(s.as_bytes());
+            let allocation = Allocation::from_bytes_byte_aligned_immutable(s.as_bytes(), ());
             let allocation = tcx.mk_const_alloc(allocation);
             ConstValue::Slice { data: allocation, meta: allocation.inner().size().bytes() }
         }
         (ast::LitKind::ByteStr(data, _), ty::Ref(_, inner_ty, _))
             if matches!(inner_ty.kind(), ty::Slice(_)) =>
         {
-            let allocation = Allocation::from_bytes_byte_aligned_immutable(data as &[u8]);
+            let allocation = Allocation::from_bytes_byte_aligned_immutable(data as &[u8], ());
             let allocation = tcx.mk_const_alloc(allocation);
             ConstValue::Slice { data: allocation, meta: allocation.inner().size().bytes() }
         }
@@ -138,7 +138,7 @@ fn lit_to_mir_constant<'tcx>(tcx: TyCtxt<'tcx>, lit_input: LitToConstInput<'tcx>
         }
         (ast::LitKind::CStr(data, _), ty::Ref(_, inner_ty, _)) if matches!(inner_ty.kind(), ty::Adt(def, _) if tcx.is_lang_item(def.did(), LangItem::CStr)) =>
         {
-            let allocation = Allocation::from_bytes_byte_aligned_immutable(data as &[u8]);
+            let allocation = Allocation::from_bytes_byte_aligned_immutable(data as &[u8], ());
             let allocation = tcx.mk_const_alloc(allocation);
             ConstValue::Slice { data: allocation, meta: allocation.inner().size().bytes() }
         }
diff --git a/compiler/rustc_mir_build/src/builder/expr/as_place.rs b/compiler/rustc_mir_build/src/builder/expr/as_place.rs
index fbe53081156..f8c64d7d13e 100644
--- a/compiler/rustc_mir_build/src/builder/expr/as_place.rs
+++ b/compiler/rustc_mir_build/src/builder/expr/as_place.rs
@@ -101,12 +101,12 @@ fn convert_to_hir_projections_and_truncate_for_capture(
                 variant = Some(*idx);
                 continue;
             }
+            ProjectionElem::UnwrapUnsafeBinder(_) => HirProjectionKind::UnwrapUnsafeBinder,
             // These do not affect anything, they just make sure we know the right type.
             ProjectionElem::OpaqueCast(_) | ProjectionElem::Subtype(..) => continue,
             ProjectionElem::Index(..)
             | ProjectionElem::ConstantIndex { .. }
-            | ProjectionElem::Subslice { .. }
-            | ProjectionElem::UnwrapUnsafeBinder(_) => {
+            | ProjectionElem::Subslice { .. } => {
                 // We don't capture array-access projections.
                 // We can stop here as arrays are captured completely.
                 break;
@@ -240,6 +240,9 @@ fn strip_prefix<'tcx>(
             HirProjectionKind::OpaqueCast => {
                 assert_matches!(iter.next(), Some(ProjectionElem::OpaqueCast(..)));
             }
+            HirProjectionKind::UnwrapUnsafeBinder => {
+                assert_matches!(iter.next(), Some(ProjectionElem::UnwrapUnsafeBinder(..)));
+            }
             HirProjectionKind::Index | HirProjectionKind::Subslice => {
                 bug!("unexpected projection kind: {:?}", projection);
             }
diff --git a/compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs b/compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs
index 5a97b08db28..b23bc089cd4 100644
--- a/compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs
+++ b/compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs
@@ -145,7 +145,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 // malloc some memory of suitable size and align:
                 let exchange_malloc = Operand::function_handle(
                     tcx,
-                    tcx.require_lang_item(LangItem::ExchangeMalloc, Some(expr_span)),
+                    tcx.require_lang_item(LangItem::ExchangeMalloc, expr_span),
                     [],
                     expr_span,
                 );
diff --git a/compiler/rustc_mir_build/src/builder/expr/into.rs b/compiler/rustc_mir_build/src/builder/expr/into.rs
index a9a07997410..2074fbce0ae 100644
--- a/compiler/rustc_mir_build/src/builder/expr/into.rs
+++ b/compiler/rustc_mir_build/src/builder/expr/into.rs
@@ -307,7 +307,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 } else if this.infcx.type_is_use_cloned_modulo_regions(this.param_env, ty) {
                     // Convert `expr.use` to a call like `Clone::clone(&expr)`
                     let success = this.cfg.start_new_block();
-                    let clone_trait = this.tcx.require_lang_item(LangItem::Clone, None);
+                    let clone_trait = this.tcx.require_lang_item(LangItem::Clone, span);
                     let clone_fn = this.tcx.associated_item_def_ids(clone_trait)[0];
                     let func = Operand::function_handle(this.tcx, clone_fn, [ty.into()], expr_span);
                     let ref_ty = Ty::new_imm_ref(this.tcx, this.tcx.lifetimes.re_erased, ty);
diff --git a/compiler/rustc_mir_build/src/builder/matches/test.rs b/compiler/rustc_mir_build/src/builder/matches/test.rs
index 210b9cce581..a4609a6053e 100644
--- a/compiler/rustc_mir_build/src/builder/matches/test.rs
+++ b/compiler/rustc_mir_build/src/builder/matches/test.rs
@@ -364,7 +364,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let borrow_kind = super::util::ref_pat_borrow_kind(mutability);
         let source_info = self.source_info(span);
         let re_erased = self.tcx.lifetimes.re_erased;
-        let trait_item = self.tcx.require_lang_item(trait_item, None);
+        let trait_item = self.tcx.require_lang_item(trait_item, span);
         let method = trait_method(self.tcx, trait_item, method, [ty]);
         let ref_src = self.temp(Ty::new_ref(self.tcx, re_erased, ty, mutability), span);
         // `let ref_src = &src_place;`
@@ -437,7 +437,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         val: Operand<'tcx>,
     ) {
         let str_ty = self.tcx.types.str_;
-        let eq_def_id = self.tcx.require_lang_item(LangItem::PartialEq, Some(source_info.span));
+        let eq_def_id = self.tcx.require_lang_item(LangItem::PartialEq, source_info.span);
         let method = trait_method(self.tcx, eq_def_id, sym::eq, [str_ty, str_ty]);
 
         let bool_ty = self.tcx.types.bool;
diff --git a/compiler/rustc_mir_build/src/builder/mod.rs b/compiler/rustc_mir_build/src/builder/mod.rs
index 9cf051a8760..3d5f6f4cf45 100644
--- a/compiler/rustc_mir_build/src/builder/mod.rs
+++ b/compiler/rustc_mir_build/src/builder/mod.rs
@@ -66,8 +66,7 @@ pub fn build_mir<'tcx>(tcx: TyCtxt<'tcx>, def: LocalDefId) -> Body<'tcx> {
                 }
             };
 
-            // this must run before MIR dump, because
-            // "not all control paths return a value" is reported here.
+            // Checking liveness after building the THIR ensures there were no typeck errors.
             //
             // maybe move the check to a MIR pass?
             tcx.ensure_ok().check_liveness(def);
@@ -451,10 +450,6 @@ fn construct_fn<'tcx>(
     let span = tcx.def_span(fn_def);
     let fn_id = tcx.local_def_id_to_hir_id(fn_def);
 
-    // The representation of thir for `-Zunpretty=thir-tree` relies on
-    // the entry expression being the last element of `thir.exprs`.
-    assert_eq!(expr.as_usize(), thir.exprs.len() - 1);
-
     // Figure out what primary body this item has.
     let body = tcx.hir_body_owned_by(fn_def);
     let span_with_body = tcx.hir_span_with_body(fn_id);
@@ -563,7 +558,7 @@ fn construct_const<'a, 'tcx>(
     // Figure out what primary body this item has.
     let (span, const_ty_span) = match tcx.hir_node(hir_id) {
         Node::Item(hir::Item {
-            kind: hir::ItemKind::Static(_, ty, _, _) | hir::ItemKind::Const(_, ty, _, _),
+            kind: hir::ItemKind::Static(_, _, ty, _) | hir::ItemKind::Const(_, _, ty, _),
             span,
             ..
         })
diff --git a/compiler/rustc_mir_build/src/builder/scope.rs b/compiler/rustc_mir_build/src/builder/scope.rs
index 2a30777e98c..67988f1fcbc 100644
--- a/compiler/rustc_mir_build/src/builder/scope.rs
+++ b/compiler/rustc_mir_build/src/builder/scope.rs
@@ -209,7 +209,7 @@ const ROOT_NODE: DropIdx = DropIdx::ZERO;
 #[derive(Debug)]
 struct DropTree {
     /// Nodes in the drop tree, containing drop data and a link to the next node.
-    drops: IndexVec<DropIdx, DropNode>,
+    drop_nodes: IndexVec<DropIdx, DropNode>,
     /// Map for finding the index of an existing node, given its contents.
     existing_drops_map: FxHashMap<DropNodeKey, DropIdx>,
     /// Edges into the `DropTree` that need to be added once it's lowered.
@@ -230,7 +230,6 @@ struct DropNode {
 struct DropNodeKey {
     next: DropIdx,
     local: Local,
-    kind: DropKind,
 }
 
 impl Scope {
@@ -278,8 +277,8 @@ impl DropTree {
         let fake_source_info = SourceInfo::outermost(DUMMY_SP);
         let fake_data =
             DropData { source_info: fake_source_info, local: Local::MAX, kind: DropKind::Storage };
-        let drops = IndexVec::from_raw(vec![DropNode { data: fake_data, next: DropIdx::MAX }]);
-        Self { drops, entry_points: Vec::new(), existing_drops_map: FxHashMap::default() }
+        let drop_nodes = IndexVec::from_raw(vec![DropNode { data: fake_data, next: DropIdx::MAX }]);
+        Self { drop_nodes, entry_points: Vec::new(), existing_drops_map: FxHashMap::default() }
     }
 
     /// Adds a node to the drop tree, consisting of drop data and the index of
@@ -288,12 +287,12 @@ impl DropTree {
     /// If there is already an equivalent node in the tree, nothing is added, and
     /// that node's index is returned. Otherwise, the new node's index is returned.
     fn add_drop(&mut self, data: DropData, next: DropIdx) -> DropIdx {
-        let drops = &mut self.drops;
+        let drop_nodes = &mut self.drop_nodes;
         *self
             .existing_drops_map
-            .entry(DropNodeKey { next, local: data.local, kind: data.kind })
+            .entry(DropNodeKey { next, local: data.local })
             // Create a new node, and also add its index to the map.
-            .or_insert_with(|| drops.push(DropNode { data, next }))
+            .or_insert_with(|| drop_nodes.push(DropNode { data, next }))
     }
 
     /// Registers `from` as an entry point to this drop tree, at `to`.
@@ -301,7 +300,7 @@ impl DropTree {
     /// During [`Self::build_mir`], `from` will be linked to the corresponding
     /// block within the drop tree.
     fn add_entry_point(&mut self, from: BasicBlock, to: DropIdx) {
-        debug_assert!(to < self.drops.next_index());
+        debug_assert!(to < self.drop_nodes.next_index());
         self.entry_points.push((to, from));
     }
 
@@ -341,10 +340,10 @@ impl DropTree {
             Own,
         }
 
-        let mut blocks = IndexVec::from_elem(None, &self.drops);
+        let mut blocks = IndexVec::from_elem(None, &self.drop_nodes);
         blocks[ROOT_NODE] = root_node;
 
-        let mut needs_block = IndexVec::from_elem(Block::None, &self.drops);
+        let mut needs_block = IndexVec::from_elem(Block::None, &self.drop_nodes);
         if root_node.is_some() {
             // In some cases (such as drops for `continue`) the root node
             // already has a block. In this case, make sure that we don't
@@ -356,7 +355,7 @@ impl DropTree {
         let entry_points = &mut self.entry_points;
         entry_points.sort();
 
-        for (drop_idx, drop_node) in self.drops.iter_enumerated().rev() {
+        for (drop_idx, drop_node) in self.drop_nodes.iter_enumerated().rev() {
             if entry_points.last().is_some_and(|entry_point| entry_point.0 == drop_idx) {
                 let block = *blocks[drop_idx].get_or_insert_with(|| T::make_block(cfg));
                 needs_block[drop_idx] = Block::Own;
@@ -396,7 +395,7 @@ impl DropTree {
         cfg: &mut CFG<'tcx>,
         blocks: &IndexSlice<DropIdx, Option<BasicBlock>>,
     ) {
-        for (drop_idx, drop_node) in self.drops.iter_enumerated().rev() {
+        for (drop_idx, drop_node) in self.drop_nodes.iter_enumerated().rev() {
             let Some(block) = blocks[drop_idx] else { continue };
             match drop_node.data.kind {
                 DropKind::Value => {
@@ -726,11 +725,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             drops
         };
 
-        let drop_idx = self.scopes.scopes[scope_index + 1..]
-            .iter()
-            .flat_map(|scope| &scope.drops)
-            .fold(ROOT_NODE, |drop_idx, &drop| drops.add_drop(drop, drop_idx));
-
+        let mut drop_idx = ROOT_NODE;
+        for scope in &self.scopes.scopes[scope_index + 1..] {
+            for drop in &scope.drops {
+                drop_idx = drops.add_drop(*drop, drop_idx);
+            }
+        }
         drops.add_entry_point(block, drop_idx);
 
         // `build_drop_trees` doesn't have access to our source_info, so we
@@ -829,9 +829,15 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         // `unwind_to` should drop the value that we're about to
                         // schedule. If dropping this value panics, then we continue
                         // with the *next* value on the unwind path.
-                        debug_assert_eq!(unwind_drops.drops[unwind_to].data.local, drop_data.local);
-                        debug_assert_eq!(unwind_drops.drops[unwind_to].data.kind, drop_data.kind);
-                        unwind_to = unwind_drops.drops[unwind_to].next;
+                        debug_assert_eq!(
+                            unwind_drops.drop_nodes[unwind_to].data.local,
+                            drop_data.local
+                        );
+                        debug_assert_eq!(
+                            unwind_drops.drop_nodes[unwind_to].data.kind,
+                            drop_data.kind
+                        );
+                        unwind_to = unwind_drops.drop_nodes[unwind_to].next;
 
                         let mut unwind_entry_point = unwind_to;
 
@@ -1551,14 +1557,14 @@ where
                 //
                 // We adjust this BEFORE we create the drop (e.g., `drops[n]`)
                 // because `drops[n]` should unwind to `drops[n-1]`.
-                debug_assert_eq!(unwind_drops.drops[unwind_to].data.local, drop_data.local);
-                debug_assert_eq!(unwind_drops.drops[unwind_to].data.kind, drop_data.kind);
-                unwind_to = unwind_drops.drops[unwind_to].next;
+                debug_assert_eq!(unwind_drops.drop_nodes[unwind_to].data.local, drop_data.local);
+                debug_assert_eq!(unwind_drops.drop_nodes[unwind_to].data.kind, drop_data.kind);
+                unwind_to = unwind_drops.drop_nodes[unwind_to].next;
 
                 if let Some(idx) = dropline_to {
-                    debug_assert_eq!(coroutine_drops.drops[idx].data.local, drop_data.local);
-                    debug_assert_eq!(coroutine_drops.drops[idx].data.kind, drop_data.kind);
-                    dropline_to = Some(coroutine_drops.drops[idx].next);
+                    debug_assert_eq!(coroutine_drops.drop_nodes[idx].data.local, drop_data.local);
+                    debug_assert_eq!(coroutine_drops.drop_nodes[idx].data.kind, drop_data.kind);
+                    dropline_to = Some(coroutine_drops.drop_nodes[idx].next);
                 }
 
                 // If the operand has been moved, and we are not on an unwind
@@ -1598,9 +1604,12 @@ where
                 // cases we emit things ALSO on the unwind path, so we need to adjust
                 // `unwind_to` in that case.
                 if storage_dead_on_unwind {
-                    debug_assert_eq!(unwind_drops.drops[unwind_to].data.local, drop_data.local);
-                    debug_assert_eq!(unwind_drops.drops[unwind_to].data.kind, drop_data.kind);
-                    unwind_to = unwind_drops.drops[unwind_to].next;
+                    debug_assert_eq!(
+                        unwind_drops.drop_nodes[unwind_to].data.local,
+                        drop_data.local
+                    );
+                    debug_assert_eq!(unwind_drops.drop_nodes[unwind_to].data.kind, drop_data.kind);
+                    unwind_to = unwind_drops.drop_nodes[unwind_to].next;
                 }
 
                 // If the operand has been moved, and we are not on an unwind
@@ -1629,14 +1638,17 @@ where
                 // the storage-dead has completed, we need to adjust the `unwind_to` pointer
                 // so that any future drops we emit will not register storage-dead.
                 if storage_dead_on_unwind {
-                    debug_assert_eq!(unwind_drops.drops[unwind_to].data.local, drop_data.local);
-                    debug_assert_eq!(unwind_drops.drops[unwind_to].data.kind, drop_data.kind);
-                    unwind_to = unwind_drops.drops[unwind_to].next;
+                    debug_assert_eq!(
+                        unwind_drops.drop_nodes[unwind_to].data.local,
+                        drop_data.local
+                    );
+                    debug_assert_eq!(unwind_drops.drop_nodes[unwind_to].data.kind, drop_data.kind);
+                    unwind_to = unwind_drops.drop_nodes[unwind_to].next;
                 }
                 if let Some(idx) = dropline_to {
-                    debug_assert_eq!(coroutine_drops.drops[idx].data.local, drop_data.local);
-                    debug_assert_eq!(coroutine_drops.drops[idx].data.kind, drop_data.kind);
-                    dropline_to = Some(coroutine_drops.drops[idx].next);
+                    debug_assert_eq!(coroutine_drops.drop_nodes[idx].data.local, drop_data.local);
+                    debug_assert_eq!(coroutine_drops.drop_nodes[idx].data.kind, drop_data.kind);
+                    dropline_to = Some(coroutine_drops.drop_nodes[idx].next);
                 }
                 // Only temps and vars need their storage dead.
                 assert!(local.index() > arg_count);
@@ -1663,10 +1675,10 @@ impl<'a, 'tcx: 'a> Builder<'a, 'tcx> {
         let is_coroutine = self.coroutine.is_some();
 
         // Link the exit drop tree to unwind drop tree.
-        if drops.drops.iter().any(|drop_node| drop_node.data.kind == DropKind::Value) {
+        if drops.drop_nodes.iter().any(|drop_node| drop_node.data.kind == DropKind::Value) {
             let unwind_target = self.diverge_cleanup_target(else_scope, span);
             let mut unwind_indices = IndexVec::from_elem_n(unwind_target, 1);
-            for (drop_idx, drop_node) in drops.drops.iter_enumerated().skip(1) {
+            for (drop_idx, drop_node) in drops.drop_nodes.iter_enumerated().skip(1) {
                 match drop_node.data.kind {
                     DropKind::Storage | DropKind::ForLint => {
                         if is_coroutine {
@@ -1695,35 +1707,29 @@ impl<'a, 'tcx: 'a> Builder<'a, 'tcx> {
         }
         // Link the exit drop tree to dropline drop tree (coroutine drop path) for async drops
         if is_coroutine
-            && drops.drops.iter().any(|DropNode { data, next: _ }| {
+            && drops.drop_nodes.iter().any(|DropNode { data, next: _ }| {
                 data.kind == DropKind::Value && self.is_async_drop(data.local)
             })
         {
             let dropline_target = self.diverge_dropline_target(else_scope, span);
             let mut dropline_indices = IndexVec::from_elem_n(dropline_target, 1);
-            for (drop_idx, drop_data) in drops.drops.iter_enumerated().skip(1) {
+            for (drop_idx, drop_data) in drops.drop_nodes.iter_enumerated().skip(1) {
+                let coroutine_drop = self
+                    .scopes
+                    .coroutine_drops
+                    .add_drop(drop_data.data, dropline_indices[drop_data.next]);
                 match drop_data.data.kind {
-                    DropKind::Storage | DropKind::ForLint => {
-                        let coroutine_drop = self
-                            .scopes
-                            .coroutine_drops
-                            .add_drop(drop_data.data, dropline_indices[drop_data.next]);
-                        dropline_indices.push(coroutine_drop);
-                    }
+                    DropKind::Storage | DropKind::ForLint => {}
                     DropKind::Value => {
-                        let coroutine_drop = self
-                            .scopes
-                            .coroutine_drops
-                            .add_drop(drop_data.data, dropline_indices[drop_data.next]);
                         if self.is_async_drop(drop_data.data.local) {
                             self.scopes.coroutine_drops.add_entry_point(
                                 blocks[drop_idx].unwrap(),
                                 dropline_indices[drop_data.next],
                             );
                         }
-                        dropline_indices.push(coroutine_drop);
                     }
                 }
+                dropline_indices.push(coroutine_drop);
             }
         }
         blocks[ROOT_NODE].map(BasicBlock::unit)
@@ -1769,11 +1775,11 @@ impl<'a, 'tcx: 'a> Builder<'a, 'tcx> {
         // prevent drop elaboration from creating drop flags that would have
         // to be captured by the coroutine. I'm not sure how important this
         // optimization is, but it is here.
-        for (drop_idx, drop_node) in drops.drops.iter_enumerated() {
+        for (drop_idx, drop_node) in drops.drop_nodes.iter_enumerated() {
             if let DropKind::Value = drop_node.data.kind
                 && let Some(bb) = blocks[drop_idx]
             {
-                debug_assert!(drop_node.next < drops.drops.next_index());
+                debug_assert!(drop_node.next < drops.drop_nodes.next_index());
                 drops.entry_points.push((drop_node.next, bb));
             }
         }
diff --git a/compiler/rustc_mir_build/src/check_unsafety.rs b/compiler/rustc_mir_build/src/check_unsafety.rs
index 9d0681b19b9..d5061b71699 100644
--- a/compiler/rustc_mir_build/src/check_unsafety.rs
+++ b/compiler/rustc_mir_build/src/check_unsafety.rs
@@ -201,9 +201,14 @@ impl<'tcx> UnsafetyVisitor<'_, 'tcx> {
     /// Handle closures/coroutines/inline-consts, which is unsafecked with their parent body.
     fn visit_inner_body(&mut self, def: LocalDefId) {
         if let Ok((inner_thir, expr)) = self.tcx.thir_body(def) {
-            // Runs all other queries that depend on THIR.
+            // Run all other queries that depend on THIR.
             self.tcx.ensure_done().mir_built(def);
-            let inner_thir = &inner_thir.steal();
+            let inner_thir = if self.tcx.sess.opts.unstable_opts.no_steal_thir {
+                &inner_thir.borrow()
+            } else {
+                // We don't have other use for the THIR. Steal it to reduce memory usage.
+                &inner_thir.steal()
+            };
             let hir_context = self.tcx.local_def_id_to_hir_id(def);
             let safety_context = mem::replace(&mut self.safety_context, SafetyContext::Safe);
             let mut inner_visitor = UnsafetyVisitor {
@@ -1157,7 +1162,12 @@ pub(crate) fn check_unsafety(tcx: TyCtxt<'_>, def: LocalDefId) {
     let Ok((thir, expr)) = tcx.thir_body(def) else { return };
     // Runs all other queries that depend on THIR.
     tcx.ensure_done().mir_built(def);
-    let thir = &thir.steal();
+    let thir = if tcx.sess.opts.unstable_opts.no_steal_thir {
+        &thir.borrow()
+    } else {
+        // We don't have other use for the THIR. Steal it to reduce memory usage.
+        &thir.steal()
+    };
 
     let hir_id = tcx.local_def_id_to_hir_id(def);
     let safety_context = tcx.hir_fn_sig_by_hir_id(hir_id).map_or(SafetyContext::Safe, |fn_sig| {
diff --git a/compiler/rustc_mir_build/src/thir/cx/expr.rs b/compiler/rustc_mir_build/src/thir/cx/expr.rs
index fde23413972..3baeccf6409 100644
--- a/compiler/rustc_mir_build/src/thir/cx/expr.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/expr.rs
@@ -38,7 +38,10 @@ impl<'tcx> ThirBuildCx<'tcx> {
     }
 
     pub(crate) fn mirror_exprs(&mut self, exprs: &'tcx [hir::Expr<'tcx>]) -> Box<[ExprId]> {
-        exprs.iter().map(|expr| self.mirror_expr_inner(expr)).collect()
+        // `mirror_exprs` may also recurse deeply, so it needs protection from stack overflow.
+        // Note that we *could* forward to `mirror_expr` for that, but we can consolidate the
+        // overhead of stack growth by doing it outside the iteration.
+        ensure_sufficient_stack(|| exprs.iter().map(|expr| self.mirror_expr_inner(expr)).collect())
     }
 
     #[instrument(level = "trace", skip(self, hir_expr))]
@@ -220,7 +223,7 @@ impl<'tcx> ThirBuildCx<'tcx> {
                 });
 
                 // kind = Pin { __pointer: pointer }
-                let pin_did = self.tcx.require_lang_item(rustc_hir::LangItem::Pin, Some(span));
+                let pin_did = self.tcx.require_lang_item(rustc_hir::LangItem::Pin, span);
                 let args = self.tcx.mk_args(&[new_pin_target.into()]);
                 let kind = ExprKind::Adt(Box::new(AdtExpr {
                     adt_def: self.tcx.adt_def(pin_did),
@@ -1220,6 +1223,9 @@ impl<'tcx> ThirBuildCx<'tcx> {
                 HirProjectionKind::OpaqueCast => {
                     ExprKind::Use { source: self.thir.exprs.push(captured_place_expr) }
                 }
+                HirProjectionKind::UnwrapUnsafeBinder => ExprKind::PlaceUnwrapUnsafeBinder {
+                    source: self.thir.exprs.push(captured_place_expr),
+                },
                 HirProjectionKind::Index | HirProjectionKind::Subslice => {
                     // We don't capture these projections, so we can ignore them here
                     continue;
diff --git a/compiler/rustc_mir_build/src/thir/cx/mod.rs b/compiler/rustc_mir_build/src/thir/cx/mod.rs
index 2f593b9a0a7..24d4136c642 100644
--- a/compiler/rustc_mir_build/src/thir/cx/mod.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/mod.rs
@@ -27,8 +27,8 @@ pub(crate) fn thir_body(
     if let Some(reported) = cx.typeck_results.tainted_by_errors {
         return Err(reported);
     }
-    let expr = cx.mirror_expr(body.value);
 
+    // Lower the params before the body's expression so errors from params are shown first.
     let owner_id = tcx.local_def_id_to_hir_id(owner_def);
     if let Some(fn_decl) = tcx.hir_fn_decl_by_hir_id(owner_id) {
         let closure_env_param = cx.closure_env_param(owner_def, owner_id);
@@ -48,6 +48,7 @@ pub(crate) fn thir_body(
         }
     }
 
+    let expr = cx.mirror_expr(body.value);
     Ok((tcx.alloc_steal_thir(cx.thir), expr))
 }
 
@@ -188,7 +189,7 @@ impl<'tcx> ThirBuildCx<'tcx> {
             // C-variadic fns also have a `VaList` input that's not listed in `fn_sig`
             // (as it's created inside the body itself, not passed in from outside).
             let ty = if fn_decl.c_variadic && index == fn_decl.inputs.len() {
-                let va_list_did = self.tcx.require_lang_item(LangItem::VaList, Some(param.span));
+                let va_list_did = self.tcx.require_lang_item(LangItem::VaList, param.span);
 
                 self.tcx
                     .type_of(va_list_did)
diff --git a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
index 78583a402fe..245bd866030 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
@@ -1047,26 +1047,21 @@ fn find_fallback_pattern_typo<'tcx>(
                 let hir::ItemKind::Use(path, _) = item.kind else {
                     continue;
                 };
-                for res in &path.res {
-                    if let Res::Def(DefKind::Const, id) = res
-                        && infcx.can_eq(param_env, ty, cx.tcx.type_of(id).instantiate_identity())
-                    {
-                        if cx.tcx.visibility(id).is_accessible_from(parent, cx.tcx) {
-                            // The original const is accessible, suggest using it directly.
-                            let item_name = cx.tcx.item_name(*id);
-                            accessible.push(item_name);
-                            accessible_path.push(with_no_trimmed_paths!(cx.tcx.def_path_str(id)));
-                        } else if cx
-                            .tcx
-                            .visibility(item.owner_id)
-                            .is_accessible_from(parent, cx.tcx)
-                        {
-                            // The const is accessible only through the re-export, point at
-                            // the `use`.
-                            let ident = item.kind.ident().unwrap();
-                            imported.push(ident.name);
-                            imported_spans.push(ident.span);
-                        }
+                if let Some(value_ns) = path.res.value_ns
+                    && let Res::Def(DefKind::Const, id) = value_ns
+                    && infcx.can_eq(param_env, ty, cx.tcx.type_of(id).instantiate_identity())
+                {
+                    if cx.tcx.visibility(id).is_accessible_from(parent, cx.tcx) {
+                        // The original const is accessible, suggest using it directly.
+                        let item_name = cx.tcx.item_name(id);
+                        accessible.push(item_name);
+                        accessible_path.push(with_no_trimmed_paths!(cx.tcx.def_path_str(id)));
+                    } else if cx.tcx.visibility(item.owner_id).is_accessible_from(parent, cx.tcx) {
+                        // The const is accessible only through the re-export, point at
+                        // the `use`.
+                        let ident = item.kind.ident().unwrap();
+                        imported.push(ident.name);
+                        imported_spans.push(ident.span);
                     }
                 }
             }
diff --git a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
index b7d203e3cd7..003ad170861 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
@@ -15,7 +15,7 @@ use rustc_middle::ty::{
 };
 use rustc_middle::{mir, span_bug};
 use rustc_span::def_id::DefId;
-use rustc_span::{Span, sym};
+use rustc_span::{DUMMY_SP, Span, sym};
 use rustc_trait_selection::traits::ObligationCause;
 use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt;
 use tracing::{debug, instrument, trace};
@@ -131,7 +131,7 @@ impl<'tcx> ConstToPat<'tcx> {
                     .dcx()
                     .create_err(ConstPatternDependsOnGenericParameter { span: self.span });
                 for arg in uv.args {
-                    if let ty::GenericArgKind::Type(ty) = arg.unpack()
+                    if let ty::GenericArgKind::Type(ty) = arg.kind()
                         && let ty::Param(param_ty) = ty.kind()
                     {
                         let def_id = self.tcx.hir_enclosing_body_owner(self.id);
@@ -382,6 +382,9 @@ fn extend_type_not_partial_eq<'tcx>(
         fn visit_ty(&mut self, ty: Ty<'tcx>) -> Self::Result {
             match ty.kind() {
                 ty::Dynamic(..) => return ControlFlow::Break(()),
+                // Unsafe binders never implement `PartialEq`, so avoid walking into them
+                // which would require instantiating its binder with placeholders too.
+                ty::UnsafeBinder(..) => return ControlFlow::Break(()),
                 ty::FnPtr(..) => return ControlFlow::Continue(()),
                 ty::Adt(def, _args) => {
                     let ty_def_id = def.did();
@@ -477,8 +480,9 @@ fn type_has_partial_eq_impl<'tcx>(
     // (If there isn't, then we can safely issue a hard
     // error, because that's never worked, due to compiler
     // using `PartialEq::eq` in this scenario in the past.)
-    let partial_eq_trait_id = tcx.require_lang_item(hir::LangItem::PartialEq, None);
-    let structural_partial_eq_trait_id = tcx.require_lang_item(hir::LangItem::StructuralPeq, None);
+    let partial_eq_trait_id = tcx.require_lang_item(hir::LangItem::PartialEq, DUMMY_SP);
+    let structural_partial_eq_trait_id =
+        tcx.require_lang_item(hir::LangItem::StructuralPeq, DUMMY_SP);
 
     let partial_eq_obligation = Obligation::new(
         tcx,
diff --git a/compiler/rustc_mir_build/src/thir/print.rs b/compiler/rustc_mir_build/src/thir/print.rs
index 37248941e2c..db9547a481f 100644
--- a/compiler/rustc_mir_build/src/thir/print.rs
+++ b/compiler/rustc_mir_build/src/thir/print.rs
@@ -8,10 +8,10 @@ use rustc_span::def_id::LocalDefId;
 /// Create a THIR tree for debugging.
 pub fn thir_tree(tcx: TyCtxt<'_>, owner_def: LocalDefId) -> String {
     match super::cx::thir_body(tcx, owner_def) {
-        Ok((thir, _)) => {
+        Ok((thir, expr)) => {
             let thir = thir.steal();
             let mut printer = ThirPrinter::new(&thir);
-            printer.print();
+            printer.print(expr);
             printer.into_buffer()
         }
         Err(_) => "error".into(),
@@ -58,7 +58,7 @@ impl<'a, 'tcx> ThirPrinter<'a, 'tcx> {
         }
     }
 
-    fn print(&mut self) {
+    fn print(&mut self, body_expr: ExprId) {
         print_indented!(self, "params: [", 0);
         for param in self.thir.params.iter() {
             self.print_param(param, 1);
@@ -66,8 +66,7 @@ impl<'a, 'tcx> ThirPrinter<'a, 'tcx> {
         print_indented!(self, "]", 0);
 
         print_indented!(self, "body:", 0);
-        let expr = ExprId::from_usize(self.thir.exprs.len() - 1);
-        self.print_expr(expr, 1);
+        self.print_expr(body_expr, 1);
     }
 
     fn into_buffer(self) -> String {
diff --git a/compiler/rustc_mir_transform/messages.ftl b/compiler/rustc_mir_transform/messages.ftl
index a1264471a2d..ae3062f07de 100644
--- a/compiler/rustc_mir_transform/messages.ftl
+++ b/compiler/rustc_mir_transform/messages.ftl
@@ -78,10 +78,4 @@ mir_transform_unconditional_recursion = function cannot return without recursing
 
 mir_transform_unconditional_recursion_call_site_label = recursive call site
 
-mir_transform_undefined_transmute = pointers cannot be transmuted to integers during const eval
-    .note = at compile-time, pointers do not have an integer value
-    .note2 = avoiding this restriction via `union` or raw pointers leads to compile-time undefined behavior
-    .help = for more information, see https://doc.rust-lang.org/std/mem/fn.transmute.html
-
 mir_transform_unknown_pass_name = MIR pass `{$name}` is unknown and will be ignored
-mir_transform_unnecessary_transmute = unnecessary transmute
diff --git a/compiler/rustc_mir_transform/src/check_undefined_transmutes.rs b/compiler/rustc_mir_transform/src/check_undefined_transmutes.rs
deleted file mode 100644
index daddb5dedbc..00000000000
--- a/compiler/rustc_mir_transform/src/check_undefined_transmutes.rs
+++ /dev/null
@@ -1,77 +0,0 @@
-use rustc_middle::mir::visit::Visitor;
-use rustc_middle::mir::{Body, Location, Operand, Terminator, TerminatorKind};
-use rustc_middle::ty::{AssocItem, AssocKind, TyCtxt};
-use rustc_session::lint::builtin::PTR_TO_INTEGER_TRANSMUTE_IN_CONSTS;
-use rustc_span::sym;
-
-use crate::errors;
-
-/// Check for transmutes that exhibit undefined behavior.
-/// For example, transmuting pointers to integers in a const context.
-pub(super) struct CheckUndefinedTransmutes;
-
-impl<'tcx> crate::MirLint<'tcx> for CheckUndefinedTransmutes {
-    fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) {
-        let mut checker = UndefinedTransmutesChecker { body, tcx };
-        checker.visit_body(body);
-    }
-}
-
-struct UndefinedTransmutesChecker<'a, 'tcx> {
-    body: &'a Body<'tcx>,
-    tcx: TyCtxt<'tcx>,
-}
-
-impl<'a, 'tcx> UndefinedTransmutesChecker<'a, 'tcx> {
-    // This functions checks two things:
-    // 1. `function` takes a raw pointer as input and returns an integer as output.
-    // 2. `function` is called from a const function or an associated constant.
-    //
-    // Why do we consider const functions and associated constants only?
-    //
-    // Generally, undefined behavior in const items are handled by the evaluator.
-    // But, const functions and associated constants are evaluated only when referenced.
-    // This can result in undefined behavior in a library going unnoticed until
-    // the function or constant is actually used.
-    //
-    // Therefore, we only consider const functions and associated constants here and leave
-    // other const items to be handled by the evaluator.
-    fn is_ptr_to_int_in_const(&self, function: &Operand<'tcx>) -> bool {
-        let def_id = self.body.source.def_id();
-
-        if self.tcx.is_const_fn(def_id)
-            || matches!(
-                self.tcx.opt_associated_item(def_id),
-                Some(AssocItem { kind: AssocKind::Const { .. }, .. })
-            )
-        {
-            let fn_sig = function.ty(self.body, self.tcx).fn_sig(self.tcx).skip_binder();
-            if let [input] = fn_sig.inputs() {
-                return input.is_raw_ptr() && fn_sig.output().is_integral();
-            }
-        }
-        false
-    }
-}
-
-impl<'tcx> Visitor<'tcx> for UndefinedTransmutesChecker<'_, 'tcx> {
-    // Check each block's terminator for calls to pointer to integer transmutes
-    // in const functions or associated constants and emit a lint.
-    fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) {
-        if let TerminatorKind::Call { func, .. } = &terminator.kind
-            && let Some((func_def_id, _)) = func.const_fn_def()
-            && self.tcx.is_intrinsic(func_def_id, sym::transmute)
-            && self.is_ptr_to_int_in_const(func)
-            && let Some(call_id) = self.body.source.def_id().as_local()
-        {
-            let hir_id = self.tcx.local_def_id_to_hir_id(call_id);
-            let span = self.body.source_info(location).span;
-            self.tcx.emit_node_span_lint(
-                PTR_TO_INTEGER_TRANSMUTE_IN_CONSTS,
-                hir_id,
-                span,
-                errors::UndefinedTransmute,
-            );
-        }
-    }
-}
diff --git a/compiler/rustc_mir_transform/src/check_unnecessary_transmutes.rs b/compiler/rustc_mir_transform/src/check_unnecessary_transmutes.rs
deleted file mode 100644
index 8da17a056e3..00000000000
--- a/compiler/rustc_mir_transform/src/check_unnecessary_transmutes.rs
+++ /dev/null
@@ -1,124 +0,0 @@
-use rustc_middle::mir::visit::Visitor;
-use rustc_middle::mir::{Body, Location, Operand, Terminator, TerminatorKind};
-use rustc_middle::ty::*;
-use rustc_session::lint::builtin::UNNECESSARY_TRANSMUTES;
-use rustc_span::source_map::Spanned;
-use rustc_span::{Span, sym};
-
-use crate::errors::UnnecessaryTransmute as Error;
-
-/// Check for transmutes that overlap with stdlib methods.
-/// For example, transmuting `[u8; 4]` to `u32`.
-pub(super) struct CheckUnnecessaryTransmutes;
-
-impl<'tcx> crate::MirLint<'tcx> for CheckUnnecessaryTransmutes {
-    fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) {
-        let mut checker = UnnecessaryTransmuteChecker { body, tcx };
-        checker.visit_body(body);
-    }
-}
-
-struct UnnecessaryTransmuteChecker<'a, 'tcx> {
-    body: &'a Body<'tcx>,
-    tcx: TyCtxt<'tcx>,
-}
-
-impl<'a, 'tcx> UnnecessaryTransmuteChecker<'a, 'tcx> {
-    fn is_unnecessary_transmute(
-        &self,
-        function: &Operand<'tcx>,
-        arg: String,
-        span: Span,
-    ) -> Option<Error> {
-        let fn_sig = function.ty(self.body, self.tcx).fn_sig(self.tcx).skip_binder();
-        let [input] = fn_sig.inputs() else { return None };
-
-        let err = |sugg| Error { span, sugg, help: None };
-
-        Some(match (input.kind(), fn_sig.output().kind()) {
-            // dont check the length; transmute does that for us.
-            // [u8; _] => primitive
-            (Array(t, _), Uint(_) | Float(_) | Int(_)) if *t.kind() == Uint(UintTy::U8) => Error {
-                sugg: format!("{}::from_ne_bytes({arg})", fn_sig.output()),
-                help: Some(
-                    "there's also `from_le_bytes` and `from_be_bytes` if you expect a particular byte order",
-                ),
-                span,
-            },
-            // primitive => [u8; _]
-            (Uint(_) | Float(_) | Int(_), Array(t, _)) if *t.kind() == Uint(UintTy::U8) => Error {
-                sugg: format!("{input}::to_ne_bytes({arg})"),
-                help: Some(
-                    "there's also `to_le_bytes` and `to_be_bytes` if you expect a particular byte order",
-                ),
-                span,
-            },
-            // char → u32
-            (Char, Uint(UintTy::U32)) => err(format!("u32::from({arg})")),
-            // char (→ u32) → i32
-            (Char, Int(IntTy::I32)) => err(format!("u32::from({arg}).cast_signed()")),
-            // u32 → char
-            (Uint(UintTy::U32), Char) => Error {
-                sugg: format!("char::from_u32_unchecked({arg})"),
-                help: Some("consider `char::from_u32(…).unwrap()`"),
-                span,
-            },
-            // i32 → char
-            (Int(IntTy::I32), Char) => Error {
-                sugg: format!("char::from_u32_unchecked(i32::cast_unsigned({arg}))"),
-                help: Some("consider `char::from_u32(i32::cast_unsigned(…)).unwrap()`"),
-                span,
-            },
-            // uNN → iNN
-            (Uint(ty), Int(_)) => err(format!("{}::cast_signed({arg})", ty.name_str())),
-            // iNN → uNN
-            (Int(ty), Uint(_)) => err(format!("{}::cast_unsigned({arg})", ty.name_str())),
-            // fNN → xsize
-            (Float(ty), Uint(UintTy::Usize)) => {
-                err(format!("{}::to_bits({arg}) as usize", ty.name_str()))
-            }
-            (Float(ty), Int(IntTy::Isize)) => {
-                err(format!("{}::to_bits({arg}) as isize", ty.name_str()))
-            }
-            // fNN (→ uNN) → iNN
-            (Float(ty), Int(..)) => err(format!("{}::to_bits({arg}).cast_signed()", ty.name_str())),
-            // fNN → uNN
-            (Float(ty), Uint(..)) => err(format!("{}::to_bits({arg})", ty.name_str())),
-            // xsize → fNN
-            (Uint(UintTy::Usize) | Int(IntTy::Isize), Float(ty)) => {
-                err(format!("{}::from_bits({arg} as _)", ty.name_str(),))
-            }
-            // iNN (→ uNN) → fNN
-            (Int(int_ty), Float(ty)) => err(format!(
-                "{}::from_bits({}::cast_unsigned({arg}))",
-                ty.name_str(),
-                int_ty.name_str()
-            )),
-            // uNN → fNN
-            (Uint(_), Float(ty)) => err(format!("{}::from_bits({arg})", ty.name_str())),
-            // bool → { x8 }
-            (Bool, Int(..) | Uint(..)) => err(format!("({arg}) as {}", fn_sig.output())),
-            // u8 → bool
-            (Uint(_), Bool) => err(format!("({arg} == 1)")),
-            _ => return None,
-        })
-    }
-}
-
-impl<'tcx> Visitor<'tcx> for UnnecessaryTransmuteChecker<'_, 'tcx> {
-    // Check each block's terminator for calls to pointer to integer transmutes
-    // in const functions or associated constants and emit a lint.
-    fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) {
-        if let TerminatorKind::Call { func, args, .. } = &terminator.kind
-            && let [Spanned { span: arg, .. }] = **args
-            && let Some((func_def_id, _)) = func.const_fn_def()
-            && self.tcx.is_intrinsic(func_def_id, sym::transmute)
-            && let span = self.body.source_info(location).span
-            && let Ok(snippet) = self.tcx.sess.source_map().span_to_snippet(arg)
-            && let Some(lint) = self.is_unnecessary_transmute(func, snippet, span)
-            && let Some(hir_id) = terminator.source_info.scope.lint_root(&self.body.source_scopes)
-        {
-            self.tcx.emit_node_span_lint(UNNECESSARY_TRANSMUTES, hir_id, span, lint);
-        }
-    }
-}
diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs
index cddb2f84778..d5d0d56f528 100644
--- a/compiler/rustc_mir_transform/src/coroutine.rs
+++ b/compiler/rustc_mir_transform/src/coroutine.rs
@@ -225,7 +225,7 @@ impl<'tcx> TransformVisitor<'tcx> {
             CoroutineKind::Coroutine(_) => span_bug!(body.span, "`Coroutine`s cannot be fused"),
             // `gen` continues return `None`
             CoroutineKind::Desugared(CoroutineDesugaring::Gen, _) => {
-                let option_def_id = self.tcx.require_lang_item(LangItem::Option, None);
+                let option_def_id = self.tcx.require_lang_item(LangItem::Option, body.span);
                 make_aggregate_adt(
                     option_def_id,
                     VariantIdx::ZERO,
@@ -242,7 +242,7 @@ impl<'tcx> TransformVisitor<'tcx> {
                     span: source_info.span,
                     const_: Const::Unevaluated(
                         UnevaluatedConst::new(
-                            self.tcx.require_lang_item(LangItem::AsyncGenFinished, None),
+                            self.tcx.require_lang_item(LangItem::AsyncGenFinished, body.span),
                             self.tcx.mk_args(&[yield_ty.into()]),
                         ),
                         self.old_yield_ty,
@@ -282,7 +282,7 @@ impl<'tcx> TransformVisitor<'tcx> {
         const ONE: VariantIdx = VariantIdx::from_usize(1);
         let rvalue = match self.coroutine_kind {
             CoroutineKind::Desugared(CoroutineDesugaring::Async, _) => {
-                let poll_def_id = self.tcx.require_lang_item(LangItem::Poll, None);
+                let poll_def_id = self.tcx.require_lang_item(LangItem::Poll, source_info.span);
                 let args = self.tcx.mk_args(&[self.old_ret_ty.into()]);
                 let (variant_idx, operands) = if is_return {
                     (ZERO, IndexVec::from_raw(vec![val])) // Poll::Ready(val)
@@ -292,7 +292,7 @@ impl<'tcx> TransformVisitor<'tcx> {
                 make_aggregate_adt(poll_def_id, variant_idx, args, operands)
             }
             CoroutineKind::Desugared(CoroutineDesugaring::Gen, _) => {
-                let option_def_id = self.tcx.require_lang_item(LangItem::Option, None);
+                let option_def_id = self.tcx.require_lang_item(LangItem::Option, source_info.span);
                 let args = self.tcx.mk_args(&[self.old_yield_ty.into()]);
                 let (variant_idx, operands) = if is_return {
                     (ZERO, IndexVec::new()) // None
@@ -310,7 +310,10 @@ impl<'tcx> TransformVisitor<'tcx> {
                         span: source_info.span,
                         const_: Const::Unevaluated(
                             UnevaluatedConst::new(
-                                self.tcx.require_lang_item(LangItem::AsyncGenFinished, None),
+                                self.tcx.require_lang_item(
+                                    LangItem::AsyncGenFinished,
+                                    source_info.span,
+                                ),
                                 self.tcx.mk_args(&[yield_ty.into()]),
                             ),
                             self.old_yield_ty,
@@ -323,7 +326,7 @@ impl<'tcx> TransformVisitor<'tcx> {
             }
             CoroutineKind::Coroutine(_) => {
                 let coroutine_state_def_id =
-                    self.tcx.require_lang_item(LangItem::CoroutineState, None);
+                    self.tcx.require_lang_item(LangItem::CoroutineState, source_info.span);
                 let args = self.tcx.mk_args(&[self.old_yield_ty.into(), self.old_ret_ty.into()]);
                 let variant_idx = if is_return {
                     ONE // CoroutineState::Complete(val)
@@ -496,7 +499,7 @@ fn make_coroutine_state_argument_indirect<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Bo
 fn make_coroutine_state_argument_pinned<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
     let ref_coroutine_ty = body.local_decls.raw[1].ty;
 
-    let pin_did = tcx.require_lang_item(LangItem::Pin, Some(body.span));
+    let pin_did = tcx.require_lang_item(LangItem::Pin, body.span);
     let pin_adt_ref = tcx.adt_def(pin_did);
     let args = tcx.mk_args(&[ref_coroutine_ty.into()]);
     let pin_ref_coroutine_ty = Ty::new_adt(tcx, pin_adt_ref, args);
@@ -557,7 +560,7 @@ fn transform_async_context<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> Ty
     // replace the type of the `resume` argument
     replace_resume_ty_local(tcx, body, CTX_ARG, context_mut_ref);
 
-    let get_context_def_id = tcx.require_lang_item(LangItem::GetContext, None);
+    let get_context_def_id = tcx.require_lang_item(LangItem::GetContext, body.span);
 
     for bb in body.basic_blocks.indices() {
         let bb_data = &body[bb];
@@ -618,7 +621,7 @@ fn replace_resume_ty_local<'tcx>(
     #[cfg(debug_assertions)]
     {
         if let ty::Adt(resume_ty_adt, _) = local_ty.kind() {
-            let expected_adt = tcx.adt_def(tcx.require_lang_item(LangItem::ResumeTy, None));
+            let expected_adt = tcx.adt_def(tcx.require_lang_item(LangItem::ResumeTy, body.span));
             assert_eq!(*resume_ty_adt, expected_adt);
         } else {
             panic!("expected `ResumeTy`, found `{:?}`", local_ty);
@@ -1095,7 +1098,7 @@ fn insert_term_block<'tcx>(body: &mut Body<'tcx>, kind: TerminatorKind<'tcx>) ->
 
 fn return_poll_ready_assign<'tcx>(tcx: TyCtxt<'tcx>, source_info: SourceInfo) -> Statement<'tcx> {
     // Poll::Ready(())
-    let poll_def_id = tcx.require_lang_item(LangItem::Poll, None);
+    let poll_def_id = tcx.require_lang_item(LangItem::Poll, source_info.span);
     let args = tcx.mk_args(&[tcx.types.unit.into()]);
     let val = Operand::Constant(Box::new(ConstOperand {
         span: source_info.span,
@@ -1437,7 +1440,7 @@ fn check_field_tys_sized<'tcx>(
             ),
             param_env,
             field_ty.ty,
-            tcx.require_lang_item(hir::LangItem::Sized, Some(field_ty.source_info.span)),
+            tcx.require_lang_item(hir::LangItem::Sized, field_ty.source_info.span),
         );
     }
 
@@ -1473,14 +1476,14 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform {
         let new_ret_ty = match coroutine_kind {
             CoroutineKind::Desugared(CoroutineDesugaring::Async, _) => {
                 // Compute Poll<return_ty>
-                let poll_did = tcx.require_lang_item(LangItem::Poll, None);
+                let poll_did = tcx.require_lang_item(LangItem::Poll, body.span);
                 let poll_adt_ref = tcx.adt_def(poll_did);
                 let poll_args = tcx.mk_args(&[old_ret_ty.into()]);
                 Ty::new_adt(tcx, poll_adt_ref, poll_args)
             }
             CoroutineKind::Desugared(CoroutineDesugaring::Gen, _) => {
                 // Compute Option<yield_ty>
-                let option_did = tcx.require_lang_item(LangItem::Option, None);
+                let option_did = tcx.require_lang_item(LangItem::Option, body.span);
                 let option_adt_ref = tcx.adt_def(option_did);
                 let option_args = tcx.mk_args(&[old_yield_ty.into()]);
                 Ty::new_adt(tcx, option_adt_ref, option_args)
@@ -1491,7 +1494,7 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform {
             }
             CoroutineKind::Coroutine(_) => {
                 // Compute CoroutineState<yield_ty, return_ty>
-                let state_did = tcx.require_lang_item(LangItem::CoroutineState, None);
+                let state_did = tcx.require_lang_item(LangItem::CoroutineState, body.span);
                 let state_adt_ref = tcx.adt_def(state_did);
                 let state_args = tcx.mk_args(&[old_yield_ty.into(), old_ret_ty.into()]);
                 Ty::new_adt(tcx, state_adt_ref, state_args)
diff --git a/compiler/rustc_mir_transform/src/coroutine/drop.rs b/compiler/rustc_mir_transform/src/coroutine/drop.rs
index 6b266da5a69..6021e795d21 100644
--- a/compiler/rustc_mir_transform/src/coroutine/drop.rs
+++ b/compiler/rustc_mir_transform/src/coroutine/drop.rs
@@ -42,7 +42,7 @@ fn build_poll_call<'tcx>(
     context_ref_place: &Place<'tcx>,
     unwind: UnwindAction,
 ) -> BasicBlock {
-    let poll_fn = tcx.require_lang_item(LangItem::FuturePoll, None);
+    let poll_fn = tcx.require_lang_item(LangItem::FuturePoll, DUMMY_SP);
     let poll_fn = Ty::new_fn_def(tcx, poll_fn, [fut_ty]);
     let poll_fn = Operand::Constant(Box::new(ConstOperand {
         span: DUMMY_SP,
@@ -77,11 +77,8 @@ fn build_pin_fut<'tcx>(
     let fut_ty = fut_place.ty(&body.local_decls, tcx).ty;
     let fut_ref_ty = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, fut_ty);
     let fut_ref_place = Place::from(body.local_decls.push(LocalDecl::new(fut_ref_ty, span)));
-    let pin_fut_new_unchecked_fn = Ty::new_fn_def(
-        tcx,
-        tcx.require_lang_item(LangItem::PinNewUnchecked, Some(span)),
-        [fut_ref_ty],
-    );
+    let pin_fut_new_unchecked_fn =
+        Ty::new_fn_def(tcx, tcx.require_lang_item(LangItem::PinNewUnchecked, span), [fut_ref_ty]);
     let fut_pin_ty = pin_fut_new_unchecked_fn.fn_sig(tcx).output().skip_binder();
     let fut_pin_place = Place::from(body.local_decls.push(LocalDecl::new(fut_pin_ty, span)));
     let pin_fut_new_unchecked_fn = Operand::Constant(Box::new(ConstOperand {
@@ -143,13 +140,15 @@ fn build_poll_switch<'tcx>(
     let Discr { val: poll_ready_discr, ty: poll_discr_ty } = poll_enum
         .discriminant_for_variant(
             tcx,
-            poll_enum_adt.variant_index_with_id(tcx.require_lang_item(LangItem::PollReady, None)),
+            poll_enum_adt
+                .variant_index_with_id(tcx.require_lang_item(LangItem::PollReady, DUMMY_SP)),
         )
         .unwrap();
     let poll_pending_discr = poll_enum
         .discriminant_for_variant(
             tcx,
-            poll_enum_adt.variant_index_with_id(tcx.require_lang_item(LangItem::PollPending, None)),
+            poll_enum_adt
+                .variant_index_with_id(tcx.require_lang_item(LangItem::PollPending, DUMMY_SP)),
         )
         .unwrap()
         .val;
@@ -316,16 +315,17 @@ pub(super) fn expand_async_drops<'tcx>(
         //  pending => return rv (yield)
         //  ready => *continue_bb|drop_bb*
 
+        let source_info = body[bb].terminator.as_ref().unwrap().source_info;
+
         // Compute Poll<> (aka Poll with void return)
-        let poll_adt_ref = tcx.adt_def(tcx.require_lang_item(LangItem::Poll, None));
+        let poll_adt_ref = tcx.adt_def(tcx.require_lang_item(LangItem::Poll, source_info.span));
         let poll_enum = Ty::new_adt(tcx, poll_adt_ref, tcx.mk_args(&[tcx.types.unit.into()]));
-        let poll_decl = LocalDecl::new(poll_enum, body.span);
+        let poll_decl = LocalDecl::new(poll_enum, source_info.span);
         let poll_unit_place = Place::from(body.local_decls.push(poll_decl));
 
         // First state-loop yield for mainline
         let context_ref_place =
-            Place::from(body.local_decls.push(LocalDecl::new(context_mut_ref, body.span)));
-        let source_info = body[bb].terminator.as_ref().unwrap().source_info;
+            Place::from(body.local_decls.push(LocalDecl::new(context_mut_ref, source_info.span)));
         let arg = Rvalue::Use(Operand::Move(Place::from(CTX_ARG)));
         body[bb].statements.push(Statement {
             source_info,
@@ -353,8 +353,9 @@ pub(super) fn expand_async_drops<'tcx>(
         let mut dropline_context_ref: Option<Place<'_>> = None;
         let mut dropline_call_bb: Option<BasicBlock> = None;
         if !is_dropline_bb {
-            let context_ref_place2: Place<'_> =
-                Place::from(body.local_decls.push(LocalDecl::new(context_mut_ref, body.span)));
+            let context_ref_place2: Place<'_> = Place::from(
+                body.local_decls.push(LocalDecl::new(context_mut_ref, source_info.span)),
+            );
             let drop_yield_block = insert_term_block(body, TerminatorKind::Unreachable); // `kind` replaced later to yield
             let drop_switch_block = build_poll_switch(
                 tcx,
@@ -382,12 +383,34 @@ pub(super) fn expand_async_drops<'tcx>(
             dropline_call_bb = Some(drop_call_bb);
         }
 
-        // value needed only for return-yields or gen-coroutines, so just const here
-        let value = Operand::Constant(Box::new(ConstOperand {
-            span: body.span,
-            user_ty: None,
-            const_: Const::from_bool(tcx, false),
-        }));
+        let value =
+            if matches!(coroutine_kind, CoroutineKind::Desugared(CoroutineDesugaring::AsyncGen, _))
+            {
+                // For AsyncGen we need `yield Poll<OptRet>::Pending`
+                let full_yield_ty = body.yield_ty().unwrap();
+                let ty::Adt(_poll_adt, args) = *full_yield_ty.kind() else { bug!() };
+                let ty::Adt(_option_adt, args) = *args.type_at(0).kind() else { bug!() };
+                let yield_ty = args.type_at(0);
+                Operand::Constant(Box::new(ConstOperand {
+                    span: source_info.span,
+                    const_: Const::Unevaluated(
+                        UnevaluatedConst::new(
+                            tcx.require_lang_item(LangItem::AsyncGenPending, source_info.span),
+                            tcx.mk_args(&[yield_ty.into()]),
+                        ),
+                        full_yield_ty,
+                    ),
+                    user_ty: None,
+                }))
+            } else {
+                // value needed only for return-yields or gen-coroutines, so just const here
+                Operand::Constant(Box::new(ConstOperand {
+                    span: source_info.span,
+                    user_ty: None,
+                    const_: Const::from_bool(tcx, false),
+                }))
+            };
+
         use rustc_middle::mir::AssertKind::ResumedAfterDrop;
         let panic_bb = insert_panic_block(tcx, body, ResumedAfterDrop(coroutine_kind));
 
@@ -573,7 +596,7 @@ pub(super) fn create_coroutine_drop_shim<'tcx>(
 
     // Update the body's def to become the drop glue.
     let coroutine_instance = body.source.instance;
-    let drop_in_place = tcx.require_lang_item(LangItem::DropInPlace, None);
+    let drop_in_place = tcx.require_lang_item(LangItem::DropInPlace, body.span);
     let drop_instance = InstanceKind::DropGlue(drop_in_place, Some(coroutine_ty));
 
     // Temporary change MirSource to coroutine's instance so that dump_mir produces more sensible
@@ -644,7 +667,7 @@ pub(super) fn create_coroutine_drop_shim_async<'tcx>(
     }
 
     // Replace the return variable: Poll<RetT> to Poll<()>
-    let poll_adt_ref = tcx.adt_def(tcx.require_lang_item(LangItem::Poll, None));
+    let poll_adt_ref = tcx.adt_def(tcx.require_lang_item(LangItem::Poll, body.span));
     let poll_enum = Ty::new_adt(tcx, poll_adt_ref, tcx.mk_args(&[tcx.types.unit.into()]));
     body.local_decls[RETURN_PLACE] = LocalDecl::with_source_info(poll_enum, source_info);
 
@@ -695,7 +718,7 @@ pub(super) fn create_coroutine_drop_shim_proxy_async<'tcx>(
     let source_info = SourceInfo::outermost(body.span);
 
     // Replace the return variable: Poll<RetT> to Poll<()>
-    let poll_adt_ref = tcx.adt_def(tcx.require_lang_item(LangItem::Poll, None));
+    let poll_adt_ref = tcx.adt_def(tcx.require_lang_item(LangItem::Poll, body.span));
     let poll_enum = Ty::new_adt(tcx, poll_adt_ref, tcx.mk_args(&[tcx.types.unit.into()]));
     body.local_decls[RETURN_PLACE] = LocalDecl::with_source_info(poll_enum, source_info);
 
diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs
index ddeae093df5..ec76076020e 100644
--- a/compiler/rustc_mir_transform/src/coverage/spans.rs
+++ b/compiler/rustc_mir_transform/src/coverage/spans.rs
@@ -1,8 +1,7 @@
 use rustc_data_structures::fx::FxHashSet;
 use rustc_middle::mir;
 use rustc_middle::ty::TyCtxt;
-use rustc_span::source_map::SourceMap;
-use rustc_span::{BytePos, DesugaringKind, ExpnKind, MacroKind, Span};
+use rustc_span::{DesugaringKind, ExpnKind, MacroKind, Span};
 use tracing::instrument;
 
 use crate::coverage::graph::{BasicCoverageBlock, CoverageGraph};
@@ -84,18 +83,8 @@ pub(super) fn extract_refined_covspans<'tcx>(
     // Discard any span that overlaps with a hole.
     discard_spans_overlapping_holes(&mut covspans, &holes);
 
-    // Discard spans that overlap in unwanted ways.
+    // Perform more refinement steps after holes have been dealt with.
     let mut covspans = remove_unwanted_overlapping_spans(covspans);
-
-    // For all empty spans, either enlarge them to be non-empty, or discard them.
-    let source_map = tcx.sess.source_map();
-    covspans.retain_mut(|covspan| {
-        let Some(span) = ensure_non_empty_span(source_map, covspan.span) else { return false };
-        covspan.span = span;
-        true
-    });
-
-    // Merge covspans that can be merged.
     covspans.dedup_by(|b, a| a.merge_if_eligible(b));
 
     code_mappings.extend(covspans.into_iter().map(|Covspan { span, bcb }| {
@@ -241,26 +230,3 @@ fn compare_spans(a: Span, b: Span) -> std::cmp::Ordering {
         // - Both have the same start and span A extends further right
         .then_with(|| Ord::cmp(&a.hi(), &b.hi()).reverse())
 }
-
-fn ensure_non_empty_span(source_map: &SourceMap, span: Span) -> Option<Span> {
-    if !span.is_empty() {
-        return Some(span);
-    }
-
-    // The span is empty, so try to enlarge it to cover an adjacent '{' or '}'.
-    source_map
-        .span_to_source(span, |src, start, end| try {
-            // Adjusting span endpoints by `BytePos(1)` is normally a bug,
-            // but in this case we have specifically checked that the character
-            // we're skipping over is one of two specific ASCII characters, so
-            // adjusting by exactly 1 byte is correct.
-            if src.as_bytes().get(end).copied() == Some(b'{') {
-                Some(span.with_hi(span.hi() + BytePos(1)))
-            } else if start > 0 && src.as_bytes()[start - 1] == b'}' {
-                Some(span.with_lo(span.lo() - BytePos(1)))
-            } else {
-                None
-            }
-        })
-        .ok()?
-}
diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
index 99b95e7312b..0cf8142a560 100644
--- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
+++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
@@ -616,7 +616,7 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> {
             place,
             operand,
             &mut |elem, op| match elem {
-                TrackElem::Field(idx) => self.ecx.project_field(op, idx.as_usize()).discard_err(),
+                TrackElem::Field(idx) => self.ecx.project_field(op, idx).discard_err(),
                 TrackElem::Variant(idx) => self.ecx.project_downcast(op, idx).discard_err(),
                 TrackElem::Discriminant => {
                     let variant = self.ecx.read_discriminant(op).discard_err()?;
@@ -890,7 +890,8 @@ fn try_write_constant<'tcx>(
 
         ty::Tuple(elem_tys) => {
             for (i, elem) in elem_tys.iter().enumerate() {
-                let Some(field) = map.apply(place, TrackElem::Field(FieldIdx::from_usize(i))) else {
+                let i = FieldIdx::from_usize(i);
+                let Some(field) = map.apply(place, TrackElem::Field(i)) else {
                     throw_machine_stop_str!("missing field in tuple")
                 };
                 let field_dest = ecx.project_field(dest, i)?;
@@ -928,7 +929,7 @@ fn try_write_constant<'tcx>(
                 let Some(field) = map.apply(variant_place, TrackElem::Field(i)) else {
                     throw_machine_stop_str!("missing field in ADT")
                 };
-                let field_dest = ecx.project_field(&variant_dest, i.as_usize())?;
+                let field_dest = ecx.project_field(&variant_dest, i)?;
                 try_write_constant(ecx, &field_dest, field, ty, state, map)?;
             }
             ecx.write_discriminant(variant_idx, dest)?;
diff --git a/compiler/rustc_mir_transform/src/elaborate_drop.rs b/compiler/rustc_mir_transform/src/elaborate_drop.rs
index 73a58160a6a..3a5e2620b14 100644
--- a/compiler/rustc_mir_transform/src/elaborate_drop.rs
+++ b/compiler/rustc_mir_transform/src/elaborate_drop.rs
@@ -1,6 +1,7 @@
 use std::{fmt, iter, mem};
 
 use rustc_abi::{FIRST_VARIANT, FieldIdx, VariantIdx};
+use rustc_hir::def::DefKind;
 use rustc_hir::lang_items::LangItem;
 use rustc_index::Idx;
 use rustc_middle::mir::*;
@@ -234,11 +235,8 @@ where
 
         let (fut_ty, drop_fn_def_id, trait_args) = if call_destructor_only {
             // Resolving obj.<AsyncDrop::drop>()
-            let trait_ref = ty::TraitRef::new(
-                tcx,
-                tcx.require_lang_item(LangItem::AsyncDrop, Some(span)),
-                [drop_ty],
-            );
+            let trait_ref =
+                ty::TraitRef::new(tcx, tcx.require_lang_item(LangItem::AsyncDrop, span), [drop_ty]);
             let (drop_trait, trait_args) = match tcx.codegen_select_candidate(
                 ty::TypingEnv::fully_monomorphized().as_query_input(trait_ref),
             ) {
@@ -251,14 +249,47 @@ where
                     span_bug!(span, "invalid `AsyncDrop` impl_source: {:?}", impl_source);
                 }
             };
-            let drop_fn_def_id = tcx.associated_item_def_ids(drop_trait)[0];
+            // impl_item_refs may be empty if drop fn is not implemented in 'impl AsyncDrop for ...'
+            // (#140974).
+            // Such code will report error, so just generate sync drop here and return
+            let Some(drop_fn_def_id) = tcx
+                .associated_item_def_ids(drop_trait)
+                .first()
+                .and_then(|def_id| {
+                    if tcx.def_kind(def_id) == DefKind::AssocFn
+                        && tcx.check_args_compatible(*def_id, trait_args)
+                    {
+                        Some(def_id)
+                    } else {
+                        None
+                    }
+                })
+                .copied()
+            else {
+                tcx.dcx().span_delayed_bug(
+                    self.elaborator.body().span,
+                    "AsyncDrop type without correct `async fn drop(...)`.",
+                );
+                self.elaborator.patch().patch_terminator(
+                    pin_obj_bb,
+                    TerminatorKind::Drop {
+                        place,
+                        target: succ,
+                        unwind: unwind.into_action(),
+                        replace: false,
+                        drop: None,
+                        async_fut: None,
+                    },
+                );
+                return pin_obj_bb;
+            };
             let drop_fn = Ty::new_fn_def(tcx, drop_fn_def_id, trait_args);
             let sig = drop_fn.fn_sig(tcx);
             let sig = tcx.instantiate_bound_regions_with_erased(sig);
             (sig.output(), drop_fn_def_id, trait_args)
         } else {
             // Resolving async_drop_in_place<T> function for drop_ty
-            let drop_fn_def_id = tcx.require_lang_item(LangItem::AsyncDropInPlace, Some(span));
+            let drop_fn_def_id = tcx.require_lang_item(LangItem::AsyncDropInPlace, span);
             let trait_args = tcx.mk_args(&[drop_ty.into()]);
             let sig = tcx.fn_sig(drop_fn_def_id).instantiate(tcx, trait_args);
             let sig = tcx.instantiate_bound_regions_with_erased(sig);
@@ -285,7 +316,7 @@ where
         // pin_obj_place preparation
         let pin_obj_new_unchecked_fn = Ty::new_fn_def(
             tcx,
-            tcx.require_lang_item(LangItem::PinNewUnchecked, Some(span)),
+            tcx.require_lang_item(LangItem::PinNewUnchecked, span),
             [GenericArg::from(obj_ref_ty)],
         );
         let pin_obj_ty = pin_obj_new_unchecked_fn.fn_sig(tcx).output().no_bound_vars().unwrap();
@@ -318,15 +349,20 @@ where
                 bug!();
             };
             let obj_ptr_ty = Ty::new_mut_ptr(tcx, drop_ty);
-            let obj_ptr_place = Place::from(self.new_temp(obj_ptr_ty));
             let unwrap_ty = adt_def.non_enum_variant().fields[FieldIdx::ZERO].ty(tcx, adt_args);
-            let addr = Rvalue::RawPtr(
-                RawPtrKind::Mut,
-                pin_obj_place.project_deeper(
-                    &[ProjectionElem::Field(FieldIdx::ZERO, unwrap_ty), ProjectionElem::Deref],
-                    tcx,
-                ),
-            );
+            let obj_ref_place = Place::from(self.new_temp(unwrap_ty));
+            call_statements.push(self.assign(
+                obj_ref_place,
+                Rvalue::Use(Operand::Copy(tcx.mk_place_field(
+                    pin_obj_place,
+                    FieldIdx::ZERO,
+                    unwrap_ty,
+                ))),
+            ));
+
+            let obj_ptr_place = Place::from(self.new_temp(obj_ptr_ty));
+
+            let addr = Rvalue::RawPtr(RawPtrKind::Mut, tcx.mk_place_deref(obj_ref_place));
             call_statements.push(self.assign(obj_ptr_place, addr));
             obj_ptr_place
         };
@@ -898,7 +934,7 @@ where
     fn destructor_call_block_sync(&mut self, (succ, unwind): (BasicBlock, Unwind)) -> BasicBlock {
         debug!("destructor_call_block_sync({:?}, {:?})", self, succ);
         let tcx = self.tcx();
-        let drop_trait = tcx.require_lang_item(LangItem::Drop, None);
+        let drop_trait = tcx.require_lang_item(LangItem::Drop, DUMMY_SP);
         let drop_fn = tcx.associated_item_def_ids(drop_trait)[0];
         let ty = self.place_ty(self.place);
 
@@ -1251,6 +1287,23 @@ where
             }
             ty::Slice(ety) => self.drop_loop_trio_for_slice(*ety),
 
+            ty::UnsafeBinder(_) => {
+                // Unsafe binders may elaborate drops if their inner type isn't copy.
+                // This is enforced in typeck, so this should never happen.
+                self.tcx().dcx().span_delayed_bug(
+                    self.source_info.span,
+                    "open drop for unsafe binder shouldn't be encountered",
+                );
+                self.elaborator.patch().new_block(BasicBlockData {
+                    statements: vec![],
+                    terminator: Some(Terminator {
+                        source_info: self.source_info,
+                        kind: TerminatorKind::Unreachable,
+                    }),
+                    is_cleanup: self.unwind.is_cleanup(),
+                })
+            }
+
             _ => span_bug!(self.source_info.span, "open drop from non-ADT `{:?}`", ty),
         }
     }
diff --git a/compiler/rustc_mir_transform/src/errors.rs b/compiler/rustc_mir_transform/src/errors.rs
index 5b03a4987ed..cffa0183fa7 100644
--- a/compiler/rustc_mir_transform/src/errors.rs
+++ b/compiler/rustc_mir_transform/src/errors.rs
@@ -158,33 +158,6 @@ pub(crate) struct MustNotSuspendReason {
     pub reason: String,
 }
 
-pub(crate) struct UnnecessaryTransmute {
-    pub span: Span,
-    pub sugg: String,
-    pub help: Option<&'static str>,
-}
-
-// Needed for def_path_str
-impl<'a> LintDiagnostic<'a, ()> for UnnecessaryTransmute {
-    fn decorate_lint<'b>(self, diag: &'b mut rustc_errors::Diag<'a, ()>) {
-        diag.primary_message(fluent::mir_transform_unnecessary_transmute);
-        diag.span_suggestion(
-            self.span,
-            "replace this with",
-            self.sugg,
-            lint::Applicability::MachineApplicable,
-        );
-        self.help.map(|help| diag.help(help));
-    }
-}
-
-#[derive(LintDiagnostic)]
-#[diag(mir_transform_undefined_transmute)]
-#[note]
-#[note(mir_transform_note2)]
-#[help]
-pub(crate) struct UndefinedTransmute;
-
 #[derive(Diagnostic)]
 #[diag(mir_transform_force_inline)]
 #[note]
diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index 209e818e9e3..92c30d239b5 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -438,8 +438,10 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                         dest.clone()
                     };
                     for (field_index, op) in fields.into_iter().enumerate() {
-                        let field_dest =
-                            self.ecx.project_field(&variant_dest, field_index).discard_err()?;
+                        let field_dest = self
+                            .ecx
+                            .project_field(&variant_dest, FieldIdx::from_usize(field_index))
+                            .discard_err()?;
                         self.ecx.copy_op(op, &field_dest).discard_err()?;
                     }
                     self.ecx
@@ -638,6 +640,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
         place: PlaceRef<'tcx>,
         value: VnIndex,
         proj: PlaceElem<'tcx>,
+        from_non_ssa_index: &mut bool,
     ) -> Option<VnIndex> {
         let proj = match proj {
             ProjectionElem::Deref => {
@@ -682,6 +685,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             }
             ProjectionElem::Index(idx) => {
                 if let Value::Repeat(inner, _) = self.get(value) {
+                    *from_non_ssa_index |= self.locals[idx].is_none();
                     return Some(*inner);
                 }
                 let idx = self.locals[idx]?;
@@ -774,6 +778,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
 
         // Invariant: `value` holds the value up-to the `index`th projection excluded.
         let mut value = self.locals[place.local]?;
+        let mut from_non_ssa_index = false;
         for (index, proj) in place.projection.iter().enumerate() {
             if let Value::Projection(pointer, ProjectionElem::Deref) = *self.get(value)
                 && let Value::Address { place: mut pointee, kind, .. } = *self.get(pointer)
@@ -791,7 +796,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             }
 
             let base = PlaceRef { local: place.local, projection: &place.projection[..index] };
-            value = self.project(base, value, proj)?;
+            value = self.project(base, value, proj, &mut from_non_ssa_index)?;
         }
 
         if let Value::Projection(pointer, ProjectionElem::Deref) = *self.get(value)
@@ -804,6 +809,9 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
         }
         if let Some(new_local) = self.try_as_local(value, location) {
             place_ref = PlaceRef { local: new_local, projection: &[] };
+        } else if from_non_ssa_index {
+            // If access to non-SSA locals is unavoidable, bail out.
+            return None;
         }
 
         if place_ref.local != place.local || place_ref.projection.len() < place.projection.len() {
@@ -1577,7 +1585,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             // We needed to check the variant to avoid trying to read the tag
             // field from an enum where no fields have variants, since that tag
             // field isn't in the `Aggregate` from which we're getting values.
-            Some((FieldIdx::from_usize(field_idx), field_layout.ty))
+            Some((field_idx, field_layout.ty))
         } else if let ty::Adt(adt, args) = ty.kind()
             && adt.is_struct()
             && adt.repr().transparent()
diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs
index 31b361ec1a9..48db536c122 100644
--- a/compiler/rustc_mir_transform/src/jump_threading.rs
+++ b/compiler/rustc_mir_transform/src/jump_threading.rs
@@ -388,7 +388,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
             lhs,
             constant,
             &mut |elem, op| match elem {
-                TrackElem::Field(idx) => self.ecx.project_field(op, idx.as_usize()).discard_err(),
+                TrackElem::Field(idx) => self.ecx.project_field(op, idx).discard_err(),
                 TrackElem::Variant(idx) => self.ecx.project_downcast(op, idx).discard_err(),
                 TrackElem::Discriminant => {
                     let variant = self.ecx.read_discriminant(op).discard_err()?;
diff --git a/compiler/rustc_mir_transform/src/large_enums.rs b/compiler/rustc_mir_transform/src/large_enums.rs
index 47cb478fe33..1a91d6bd7da 100644
--- a/compiler/rustc_mir_transform/src/large_enums.rs
+++ b/compiler/rustc_mir_transform/src/large_enums.rs
@@ -241,6 +241,7 @@ impl EnumSizeOpt {
             data,
             tcx.data_layout.ptr_sized_integer().align(&tcx.data_layout).abi,
             Mutability::Not,
+            (),
         );
         let alloc = tcx.reserve_and_set_memory_alloc(tcx.mk_const_alloc(alloc));
         Some((*adt_def, num_discrs, *alloc_cache.entry(ty).or_insert(alloc)))
diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs
index 10dbb3437dc..572ad585c8c 100644
--- a/compiler/rustc_mir_transform/src/lib.rs
+++ b/compiler/rustc_mir_transform/src/lib.rs
@@ -7,10 +7,7 @@
 #![feature(file_buffered)]
 #![feature(if_let_guard)]
 #![feature(impl_trait_in_assoc_type)]
-#![feature(map_try_insert)]
-#![feature(never_type)]
 #![feature(try_blocks)]
-#![feature(vec_deque_pop_if)]
 #![feature(yeet_expr)]
 // tidy-alphabetical-end
 
@@ -123,8 +120,6 @@ declare_passes! {
     mod check_const_item_mutation : CheckConstItemMutation;
     mod check_null : CheckNull;
     mod check_packed_ref : CheckPackedRef;
-    mod check_undefined_transmutes : CheckUndefinedTransmutes;
-    mod check_unnecessary_transmutes: CheckUnnecessaryTransmutes;
     // This pass is public to allow external drivers to perform MIR cleanup
     pub mod cleanup_post_borrowck : CleanupPostBorrowck;
 
@@ -390,8 +385,6 @@ fn mir_built(tcx: TyCtxt<'_>, def: LocalDefId) -> &Steal<Body<'_>> {
             &Lint(check_packed_ref::CheckPackedRef),
             &Lint(check_const_item_mutation::CheckConstItemMutation),
             &Lint(function_item_references::FunctionItemReferences),
-            &Lint(check_undefined_transmutes::CheckUndefinedTransmutes),
-            &Lint(check_unnecessary_transmutes::CheckUnnecessaryTransmutes),
             // What we need to do constant evaluation.
             &simplify::SimplifyCfg::Initial,
             &Lint(sanity_check::SanityCheck),
diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs
index 9c21bcfc0d2..52f4c39c09b 100644
--- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs
+++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs
@@ -262,6 +262,52 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
                         });
                         terminator.kind = TerminatorKind::Goto { target };
                     }
+                    sym::slice_get_unchecked => {
+                        let target = target.unwrap();
+                        let Ok([ptrish, index]) = take_array(args) else {
+                            span_bug!(
+                                terminator.source_info.span,
+                                "Wrong number of arguments for {intrinsic:?}",
+                            );
+                        };
+
+                        let place = ptrish.node.place().unwrap();
+                        assert!(!place.is_indirect());
+                        let updated_place = place.project_deeper(
+                            &[
+                                ProjectionElem::Deref,
+                                ProjectionElem::Index(
+                                    index.node.place().unwrap().as_local().unwrap(),
+                                ),
+                            ],
+                            tcx,
+                        );
+
+                        let ret_ty = generic_args.type_at(0);
+                        let rvalue = match *ret_ty.kind() {
+                            ty::RawPtr(_, Mutability::Not) => {
+                                Rvalue::RawPtr(RawPtrKind::Const, updated_place)
+                            }
+                            ty::RawPtr(_, Mutability::Mut) => {
+                                Rvalue::RawPtr(RawPtrKind::Mut, updated_place)
+                            }
+                            ty::Ref(region, _, Mutability::Not) => {
+                                Rvalue::Ref(region, BorrowKind::Shared, updated_place)
+                            }
+                            ty::Ref(region, _, Mutability::Mut) => Rvalue::Ref(
+                                region,
+                                BorrowKind::Mut { kind: MutBorrowKind::Default },
+                                updated_place,
+                            ),
+                            _ => bug!("Unknown return type {ret_ty:?}"),
+                        };
+
+                        block.statements.push(Statement {
+                            source_info: terminator.source_info,
+                            kind: StatementKind::Assign(Box::new((*destination, rvalue))),
+                        });
+                        terminator.kind = TerminatorKind::Goto { target };
+                    }
                     sym::transmute | sym::transmute_unchecked => {
                         let dst_ty = destination.ty(local_decls, tcx).ty;
                         let Ok([arg]) = take_array(args) else {
diff --git a/compiler/rustc_mir_transform/src/match_branches.rs b/compiler/rustc_mir_transform/src/match_branches.rs
index 8c0c3096899..5e511f1a418 100644
--- a/compiler/rustc_mir_transform/src/match_branches.rs
+++ b/compiler/rustc_mir_transform/src/match_branches.rs
@@ -284,12 +284,14 @@ fn can_cast(
     let v = match src_layout.ty.kind() {
         ty::Uint(_) => from_scalar.to_uint(src_layout.size),
         ty::Int(_) => from_scalar.to_int(src_layout.size) as u128,
-        _ => unreachable!("invalid int"),
+        // We can also transform the values of other integer representations (such as char),
+        // although this may not be practical in real-world scenarios.
+        _ => return false,
     };
     let size = match *cast_ty.kind() {
         ty::Int(t) => Integer::from_int_ty(&tcx, t).size(),
         ty::Uint(t) => Integer::from_uint_ty(&tcx, t).size(),
-        _ => unreachable!("invalid int"),
+        _ => return false,
     };
     let v = size.truncate(v);
     let cast_scalar = ScalarInt::try_from_uint(v, size).unwrap();
diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs
index 9688ac8ed2e..6d45bbc6e16 100644
--- a/compiler/rustc_mir_transform/src/shim.rs
+++ b/compiler/rustc_mir_transform/src/shim.rs
@@ -98,7 +98,7 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body<
             build_call_shim(tcx, instance, None, CallKind::Direct(def_id))
         }
         ty::InstanceKind::ClosureOnceShim { call_once: _, track_caller: _ } => {
-            let fn_mut = tcx.require_lang_item(LangItem::FnMut, None);
+            let fn_mut = tcx.require_lang_item(LangItem::FnMut, DUMMY_SP);
             let call_mut = tcx
                 .associated_items(fn_mut)
                 .in_definition_order()
diff --git a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs
index 7976b65aae7..fd7b7362cd9 100644
--- a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs
+++ b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs
@@ -6,7 +6,7 @@ use rustc_middle::mir::{
     BasicBlock, BasicBlockData, Body, Local, LocalDecl, MirSource, Operand, Place, Rvalue,
     SourceInfo, Statement, StatementKind, Terminator, TerminatorKind,
 };
-use rustc_middle::ty::{self, EarlyBinder, Ty, TyCtxt};
+use rustc_middle::ty::{self, EarlyBinder, Ty, TyCtxt, TypeVisitableExt};
 
 use super::*;
 use crate::patch::MirPatch;
@@ -64,7 +64,7 @@ pub(super) fn build_async_drop_shim<'tcx>(
     let needs_async_drop = drop_ty.needs_async_drop(tcx, typing_env);
     let needs_sync_drop = !needs_async_drop && drop_ty.needs_drop(tcx, typing_env);
 
-    let resume_adt = tcx.adt_def(tcx.require_lang_item(LangItem::ResumeTy, None));
+    let resume_adt = tcx.adt_def(tcx.require_lang_item(LangItem::ResumeTy, DUMMY_SP));
     let resume_ty = Ty::new_adt(tcx, resume_adt, ty::List::empty());
 
     let fn_sig = ty::Binder::dummy(tcx.mk_fn_sig(
@@ -121,9 +121,10 @@ pub(super) fn build_async_drop_shim<'tcx>(
         parent_args.as_coroutine().resume_ty(),
     )));
     body.phase = MirPhase::Runtime(RuntimePhase::Initial);
-    if !needs_async_drop {
+    if !needs_async_drop || drop_ty.references_error() {
         // Returning noop body for types without `need async drop`
-        // (or sync Drop in case of !`need async drop` && `need drop`)
+        // (or sync Drop in case of !`need async drop` && `need drop`).
+        // And also for error types.
         return body;
     }
 
@@ -219,7 +220,7 @@ fn build_adrop_for_coroutine_shim<'tcx>(
     body.source.instance = instance;
     body.phase = MirPhase::Runtime(RuntimePhase::Initial);
     body.var_debug_info.clear();
-    let pin_adt_ref = tcx.adt_def(tcx.require_lang_item(LangItem::Pin, Some(span)));
+    let pin_adt_ref = tcx.adt_def(tcx.require_lang_item(LangItem::Pin, span));
     let args = tcx.mk_args(&[proxy_ref.into()]);
     let pin_proxy_ref = Ty::new_adt(tcx, pin_adt_ref, args);
 
@@ -307,10 +308,10 @@ fn build_adrop_for_adrop_shim<'tcx>(
     let cor_ref = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, impl_ty);
 
     // ret_ty = `Poll<()>`
-    let poll_adt_ref = tcx.adt_def(tcx.require_lang_item(LangItem::Poll, None));
+    let poll_adt_ref = tcx.adt_def(tcx.require_lang_item(LangItem::Poll, span));
     let ret_ty = Ty::new_adt(tcx, poll_adt_ref, tcx.mk_args(&[tcx.types.unit.into()]));
     // env_ty = `Pin<&mut proxy_ty>`
-    let pin_adt_ref = tcx.adt_def(tcx.require_lang_item(LangItem::Pin, None));
+    let pin_adt_ref = tcx.adt_def(tcx.require_lang_item(LangItem::Pin, span));
     let env_ty = Ty::new_adt(tcx, pin_adt_ref, tcx.mk_args(&[proxy_ref.into()]));
     // sig = `fn (Pin<&mut proxy_ty>, &mut Context) -> Poll<()>`
     let sig = tcx.mk_fn_sig(
@@ -375,7 +376,7 @@ fn build_adrop_for_adrop_shim<'tcx>(
     let cor_pin_ty = Ty::new_adt(tcx, pin_adt_ref, tcx.mk_args(&[cor_ref.into()]));
     let cor_pin_place = Place::from(locals.push(LocalDecl::new(cor_pin_ty, span)));
 
-    let pin_fn = tcx.require_lang_item(LangItem::PinNewUnchecked, Some(span));
+    let pin_fn = tcx.require_lang_item(LangItem::PinNewUnchecked, span);
     // call Pin<FutTy>::new_unchecked(&mut impl_cor)
     blocks.push(BasicBlockData {
         statements,
@@ -395,7 +396,7 @@ fn build_adrop_for_adrop_shim<'tcx>(
     });
     // When dropping async drop coroutine, we continue its execution:
     // we call impl::poll (impl_layout, ctx)
-    let poll_fn = tcx.require_lang_item(LangItem::FuturePoll, None);
+    let poll_fn = tcx.require_lang_item(LangItem::FuturePoll, span);
     let resume_ctx = Place::from(Local::new(2));
     blocks.push(BasicBlockData {
         statements: vec![],
diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs
index c8aa7588d03..fd91508cc11 100644
--- a/compiler/rustc_mir_transform/src/validate.rs
+++ b/compiler/rustc_mir_transform/src/validate.rs
@@ -1253,7 +1253,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                             self.tcx,
                             self.tcx.require_lang_item(
                                 LangItem::CoerceUnsized,
-                                Some(self.body.source_info(location).span),
+                                self.body.source_info(location).span,
                             ),
                             [op_ty, *target_type],
                         )) {
diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs
index b3d7eaf332b..173030e0326 100644
--- a/compiler/rustc_monomorphize/src/collector.rs
+++ b/compiler/rustc_monomorphize/src/collector.rs
@@ -781,7 +781,7 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirUsedCollector<'a, 'tcx> {
 
         let tcx = self.tcx;
         let push_mono_lang_item = |this: &mut Self, lang_item: LangItem| {
-            let instance = Instance::mono(tcx, tcx.require_lang_item(lang_item, Some(source)));
+            let instance = Instance::mono(tcx, tcx.require_lang_item(lang_item, source));
             if tcx.should_codegen_locally(instance) {
                 this.used_items.push(create_fn_mono_item(tcx, instance, source));
             }
@@ -921,7 +921,7 @@ fn visit_instance_use<'tcx>(
             // be lowered in codegen to nothing or a call to panic_nounwind. So if we encounter any
             // of those intrinsics, we need to include a mono item for panic_nounwind, else we may try to
             // codegen a call to that function without generating code for the function itself.
-            let def_id = tcx.require_lang_item(LangItem::PanicNounwind, None);
+            let def_id = tcx.require_lang_item(LangItem::PanicNounwind, source);
             let panic_instance = Instance::mono(tcx, def_id);
             if tcx.should_codegen_locally(panic_instance) {
                 output.push(create_fn_mono_item(tcx, panic_instance, source));
@@ -1483,7 +1483,7 @@ impl<'v> RootCollector<'_, 'v> {
 
                 // But even just declaring them must collect the items they refer to
                 // unless their generics require monomorphization.
-                if !self.tcx.generics_of(id.owner_id).requires_monomorphization(self.tcx)
+                if !self.tcx.generics_of(id.owner_id).own_requires_monomorphization()
                     && let Ok(val) = self.tcx.const_eval_poly(id.owner_id.to_def_id())
                 {
                     collect_const_value(self.tcx, val, self.output);
diff --git a/compiler/rustc_monomorphize/src/lib.rs b/compiler/rustc_monomorphize/src/lib.rs
index 5c66017bc61..05683940cba 100644
--- a/compiler/rustc_monomorphize/src/lib.rs
+++ b/compiler/rustc_monomorphize/src/lib.rs
@@ -29,7 +29,7 @@ fn custom_coerce_unsize_info<'tcx>(
 ) -> Result<CustomCoerceUnsized, ErrorGuaranteed> {
     let trait_ref = ty::TraitRef::new(
         tcx.tcx,
-        tcx.require_lang_item(LangItem::CoerceUnsized, Some(tcx.span)),
+        tcx.require_lang_item(LangItem::CoerceUnsized, tcx.span),
         [source_ty, target_ty],
     );
 
diff --git a/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs b/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs
index cfeaee07776..8dbbb4d1713 100644
--- a/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs
+++ b/compiler/rustc_monomorphize/src/mono_checks/abi_check.rs
@@ -1,6 +1,6 @@
 //! This module ensures that if a function's ABI requires a particular target feature,
 //! that target feature is enabled both on the callee and all callers.
-use rustc_abi::{BackendRepr, RegKind};
+use rustc_abi::{BackendRepr, CanonAbi, RegKind, X86Call};
 use rustc_hir::{CRATE_HIR_ID, HirId};
 use rustc_middle::mir::{self, Location, traversal};
 use rustc_middle::ty::layout::LayoutCx;
@@ -8,7 +8,7 @@ use rustc_middle::ty::{self, Instance, InstanceKind, Ty, TyCtxt, TypingEnv};
 use rustc_session::lint::builtin::WASM_C_ABI;
 use rustc_span::def_id::DefId;
 use rustc_span::{DUMMY_SP, Span, Symbol, sym};
-use rustc_target::callconv::{ArgAbi, Conv, FnAbi, PassMode};
+use rustc_target::callconv::{ArgAbi, FnAbi, PassMode};
 use rustc_target::spec::{HasWasmCAbiOpt, WasmCAbi};
 
 use crate::errors;
@@ -35,8 +35,6 @@ fn do_check_simd_vector_abi<'tcx>(
     is_call: bool,
     loc: impl Fn() -> (Span, HirId),
 ) {
-    // We check this on all functions, including those using the "Rust" ABI.
-    // For the "Rust" ABI it would be a bug if the lint ever triggered, but better safe than sorry.
     let feature_def = tcx.sess.target.features_for_correct_vector_abi();
     let codegen_attrs = tcx.codegen_fn_attrs(def_id);
     let have_feature = |feat: Symbol| {
@@ -72,7 +70,7 @@ fn do_check_simd_vector_abi<'tcx>(
         }
     }
     // The `vectorcall` ABI is special in that it requires SSE2 no matter which types are being passed.
-    if abi.conv == Conv::X86VectorCall && !have_feature(sym::sse2) {
+    if abi.conv == CanonAbi::X86(X86Call::Vectorcall) && !have_feature(sym::sse2) {
         let (span, _hir_id) = loc();
         tcx.dcx().emit_err(errors::AbiRequiredTargetFeature {
             span,
@@ -123,12 +121,13 @@ fn do_check_wasm_abi<'tcx>(
     is_call: bool,
     loc: impl Fn() -> (Span, HirId),
 ) {
-    // Only proceed for `extern "C" fn` on wasm32-unknown-unknown (same check as what `adjust_for_foreign_abi` uses to call `compute_wasm_abi_info`),
-    // and only proceed if `wasm_c_abi_opt` indicates we should emit the lint.
+    // Only proceed for `extern "C" fn` on wasm32-unknown-unknown (same check as what
+    // `adjust_for_foreign_abi` uses to call `compute_wasm_abi_info`), and only proceed if
+    // `wasm_c_abi_opt` indicates we should emit the lint.
     if !(tcx.sess.target.arch == "wasm32"
         && tcx.sess.target.os == "unknown"
         && tcx.wasm_c_abi_opt() == WasmCAbi::Legacy { with_lint: true }
-        && abi.conv == Conv::C)
+        && abi.conv == CanonAbi::C)
     {
         return;
     }
@@ -157,8 +156,15 @@ fn check_instance_abi<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) {
     else {
         // An error will be reported during codegen if we cannot determine the ABI of this
         // function.
+        tcx.dcx().delayed_bug("ABI computation failure should lead to compilation failure");
         return;
     };
+    // Unlike the call-site check, we do also check "Rust" ABI functions here.
+    // This should never trigger, *except* if we start making use of vector registers
+    // for the "Rust" ABI and the user disables those vector registers (which should trigger a
+    // warning as that's clearly disabling a "required" target feature for this target).
+    // Using such a function is where disabling the vector register actually can start leading
+    // to soundness issues, so erroring here seems good.
     let loc = || {
         let def_id = instance.def_id();
         (
@@ -179,7 +185,8 @@ fn check_call_site_abi<'tcx>(
     loc: impl Fn() -> (Span, HirId) + Copy,
 ) {
     if callee.fn_sig(tcx).abi().is_rustic_abi() {
-        // we directly handle the soundness of Rust ABIs
+        // We directly handle the soundness of Rust ABIs -- so let's skip the majority of
+        // call sites to avoid a perf regression.
         return;
     }
     let typing_env = ty::TypingEnv::fully_monomorphized();
diff --git a/compiler/rustc_monomorphize/src/partitioning.rs b/compiler/rustc_monomorphize/src/partitioning.rs
index b4169a060d4..49025673bbd 100644
--- a/compiler/rustc_monomorphize/src/partitioning.rs
+++ b/compiler/rustc_monomorphize/src/partitioning.rs
@@ -461,15 +461,15 @@ fn merge_codegen_units<'tcx>(
 
         for cgu in codegen_units.iter_mut() {
             if let Some(new_cgu_name) = new_cgu_names.get(&cgu.name()) {
-                if cx.tcx.sess.opts.unstable_opts.human_readable_cgu_names {
-                    cgu.set_name(Symbol::intern(new_cgu_name));
+                let new_cgu_name = if cx.tcx.sess.opts.unstable_opts.human_readable_cgu_names {
+                    Symbol::intern(&CodegenUnit::shorten_name(new_cgu_name))
                 } else {
                     // If we don't require CGU names to be human-readable,
                     // we use a fixed length hash of the composite CGU name
                     // instead.
-                    let new_cgu_name = CodegenUnit::mangle_name(new_cgu_name);
-                    cgu.set_name(Symbol::intern(&new_cgu_name));
-                }
+                    Symbol::intern(&CodegenUnit::mangle_name(new_cgu_name))
+                };
+                cgu.set_name(new_cgu_name);
             }
         }
 
diff --git a/compiler/rustc_next_trait_solver/src/canonicalizer.rs b/compiler/rustc_next_trait_solver/src/canonicalizer.rs
index bbb4a162027..a87ae5284b1 100644
--- a/compiler/rustc_next_trait_solver/src/canonicalizer.rs
+++ b/compiler/rustc_next_trait_solver/src/canonicalizer.rs
@@ -4,12 +4,22 @@ use rustc_type_ir::data_structures::{HashMap, ensure_sufficient_stack};
 use rustc_type_ir::inherent::*;
 use rustc_type_ir::solve::{Goal, QueryInput};
 use rustc_type_ir::{
-    self as ty, Canonical, CanonicalTyVarKind, CanonicalVarInfo, CanonicalVarKind, InferCtxtLike,
-    Interner, TypeFoldable, TypeFolder, TypeSuperFoldable, TypeVisitableExt,
+    self as ty, Canonical, CanonicalTyVarKind, CanonicalVarKind, Flags, InferCtxtLike, Interner,
+    TypeFlags, TypeFoldable, TypeFolder, TypeSuperFoldable, TypeVisitableExt,
 };
 
 use crate::delegate::SolverDelegate;
 
+/// Does this have infer/placeholder/param, free regions or ReErased?
+const NEEDS_CANONICAL: TypeFlags = TypeFlags::from_bits(
+    TypeFlags::HAS_INFER.bits()
+        | TypeFlags::HAS_PLACEHOLDER.bits()
+        | TypeFlags::HAS_PARAM.bits()
+        | TypeFlags::HAS_FREE_REGIONS.bits()
+        | TypeFlags::HAS_RE_ERASED.bits(),
+)
+.unwrap();
+
 /// Whether we're canonicalizing a query input or the query response.
 ///
 /// When canonicalizing an input we're in the context of the caller
@@ -50,7 +60,7 @@ pub struct Canonicalizer<'a, D: SolverDelegate<Interner = I>, I: Interner> {
 
     // Mutable fields.
     variables: &'a mut Vec<I::GenericArg>,
-    primitive_var_infos: Vec<CanonicalVarInfo<I>>,
+    var_kinds: Vec<CanonicalVarKind<I>>,
     variable_lookup_table: HashMap<I::GenericArg, usize>,
     binder_index: ty::DebruijnIndex,
 
@@ -73,15 +83,19 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
 
             variables,
             variable_lookup_table: Default::default(),
-            primitive_var_infos: Vec::new(),
+            var_kinds: Vec::new(),
             binder_index: ty::INNERMOST,
 
             cache: Default::default(),
         };
 
-        let value = value.fold_with(&mut canonicalizer);
-        assert!(!value.has_infer(), "unexpected infer in {value:?}");
-        assert!(!value.has_placeholders(), "unexpected placeholders in {value:?}");
+        let value = if value.has_type_flags(NEEDS_CANONICAL) {
+            value.fold_with(&mut canonicalizer)
+        } else {
+            value
+        };
+        debug_assert!(!value.has_infer(), "unexpected infer in {value:?}");
+        debug_assert!(!value.has_placeholders(), "unexpected placeholders in {value:?}");
         let (max_universe, variables) = canonicalizer.finalize();
         Canonical { max_universe, variables, value }
     }
@@ -106,12 +120,19 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
 
             variables,
             variable_lookup_table: Default::default(),
-            primitive_var_infos: Vec::new(),
+            var_kinds: Vec::new(),
             binder_index: ty::INNERMOST,
 
             cache: Default::default(),
         };
-        let param_env = input.goal.param_env.fold_with(&mut env_canonicalizer);
+
+        let param_env = input.goal.param_env;
+        let param_env = if param_env.has_type_flags(NEEDS_CANONICAL) {
+            param_env.fold_with(&mut env_canonicalizer)
+        } else {
+            param_env
+        };
+
         debug_assert_eq!(env_canonicalizer.binder_index, ty::INNERMOST);
         // Then canonicalize the rest of the input without keeping `'static`
         // while *mostly* reusing the canonicalizer from above.
@@ -123,7 +144,7 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
             // We're able to reuse the `variable_lookup_table` as whether or not
             // it already contains an entry for `'static` does not matter.
             variable_lookup_table: env_canonicalizer.variable_lookup_table,
-            primitive_var_infos: env_canonicalizer.primitive_var_infos,
+            var_kinds: env_canonicalizer.var_kinds,
             binder_index: ty::INNERMOST,
 
             // We do not reuse the cache as it may contain entries whose canonicalized
@@ -134,14 +155,26 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
             cache: Default::default(),
         };
 
-        let predicate = input.goal.predicate.fold_with(&mut rest_canonicalizer);
+        let predicate = input.goal.predicate;
+        let predicate = if predicate.has_type_flags(NEEDS_CANONICAL) {
+            predicate.fold_with(&mut rest_canonicalizer)
+        } else {
+            predicate
+        };
         let goal = Goal { param_env, predicate };
+
+        let predefined_opaques_in_body = input.predefined_opaques_in_body;
         let predefined_opaques_in_body =
-            input.predefined_opaques_in_body.fold_with(&mut rest_canonicalizer);
+            if input.predefined_opaques_in_body.has_type_flags(NEEDS_CANONICAL) {
+                predefined_opaques_in_body.fold_with(&mut rest_canonicalizer)
+            } else {
+                predefined_opaques_in_body
+            };
+
         let value = QueryInput { goal, predefined_opaques_in_body };
 
-        assert!(!value.has_infer(), "unexpected infer in {value:?}");
-        assert!(!value.has_placeholders(), "unexpected placeholders in {value:?}");
+        debug_assert!(!value.has_infer(), "unexpected infer in {value:?}");
+        debug_assert!(!value.has_placeholders(), "unexpected placeholders in {value:?}");
         let (max_universe, variables) = rest_canonicalizer.finalize();
         Canonical { max_universe, variables, value }
     }
@@ -149,7 +182,7 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
     fn get_or_insert_bound_var(
         &mut self,
         arg: impl Into<I::GenericArg>,
-        canonical_var_info: CanonicalVarInfo<I>,
+        kind: CanonicalVarKind<I>,
     ) -> ty::BoundVar {
         // FIXME: 16 is made up and arbitrary. We should look at some
         // perf data here.
@@ -162,14 +195,14 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
             *self.variable_lookup_table.entry(arg).or_insert_with(|| {
                 let var = self.variables.len();
                 self.variables.push(arg);
-                self.primitive_var_infos.push(canonical_var_info);
+                self.var_kinds.push(kind);
                 var
             })
         } else {
             self.variables.iter().position(|&v| v == arg).unwrap_or_else(|| {
                 let var = self.variables.len();
                 self.variables.push(arg);
-                self.primitive_var_infos.push(canonical_var_info);
+                self.var_kinds.push(kind);
                 var
             })
         };
@@ -177,8 +210,8 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
         ty::BoundVar::from(idx)
     }
 
-    fn finalize(self) -> (ty::UniverseIndex, I::CanonicalVars) {
-        let mut var_infos = self.primitive_var_infos;
+    fn finalize(self) -> (ty::UniverseIndex, I::CanonicalVarKinds) {
+        let mut var_kinds = self.var_kinds;
         // See the rustc-dev-guide section about how we deal with universes
         // during canonicalization in the new solver.
         match self.canonicalize_mode {
@@ -192,25 +225,25 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
             // information for placeholders and inference variables created inside
             // of the query.
             CanonicalizeMode::Response { max_input_universe } => {
-                for var in var_infos.iter_mut() {
+                for var in var_kinds.iter_mut() {
                     let uv = var.universe();
                     let new_uv = ty::UniverseIndex::from(
                         uv.index().saturating_sub(max_input_universe.index()),
                     );
                     *var = var.with_updated_universe(new_uv);
                 }
-                let max_universe = var_infos
+                let max_universe = var_kinds
                     .iter()
-                    .map(|info| info.universe())
+                    .map(|kind| kind.universe())
                     .max()
                     .unwrap_or(ty::UniverseIndex::ROOT);
 
-                let var_infos = self.delegate.cx().mk_canonical_var_infos(&var_infos);
-                return (max_universe, var_infos);
+                let var_kinds = self.delegate.cx().mk_canonical_var_kinds(&var_kinds);
+                return (max_universe, var_kinds);
             }
         }
 
-        // Given a `var_infos` with existentials `En` and universals `Un` in
+        // Given a `var_kinds` with existentials `En` and universals `Un` in
         // universes `n`, this algorithm compresses them in place so that:
         //
         // - the new universe indices are as small as possible
@@ -219,12 +252,12 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
         //   2. put a placeholder in the same universe as an existential which cannot name it
         //
         // Let's walk through an example:
-        // - var_infos: [E0, U1, E5, U2, E2, E6, U6], curr_compressed_uv: 0, next_orig_uv: 0
-        // - var_infos: [E0, U1, E5, U2, E2, E6, U6], curr_compressed_uv: 0, next_orig_uv: 1
-        // - var_infos: [E0, U1, E5, U2, E2, E6, U6], curr_compressed_uv: 1, next_orig_uv: 2
-        // - var_infos: [E0, U1, E5, U1, E1, E6, U6], curr_compressed_uv: 1, next_orig_uv: 5
-        // - var_infos: [E0, U1, E2, U1, E1, E6, U6], curr_compressed_uv: 2, next_orig_uv: 6
-        // - var_infos: [E0, U1, E1, U1, E1, E3, U3], curr_compressed_uv: 2, next_orig_uv: -
+        // - var_kinds: [E0, U1, E5, U2, E2, E6, U6], curr_compressed_uv: 0, next_orig_uv: 0
+        // - var_kinds: [E0, U1, E5, U2, E2, E6, U6], curr_compressed_uv: 0, next_orig_uv: 1
+        // - var_kinds: [E0, U1, E5, U2, E2, E6, U6], curr_compressed_uv: 1, next_orig_uv: 2
+        // - var_kinds: [E0, U1, E5, U1, E1, E6, U6], curr_compressed_uv: 1, next_orig_uv: 5
+        // - var_kinds: [E0, U1, E2, U1, E1, E6, U6], curr_compressed_uv: 2, next_orig_uv: 6
+        // - var_kinds: [E0, U1, E1, U1, E1, E3, U3], curr_compressed_uv: 2, next_orig_uv: -
         //
         // This algorithm runs in `O(mn)` where `n` is the number of different universes and
         // `m` the number of variables. This should be fine as both are expected to be small.
@@ -232,7 +265,7 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
         let mut existential_in_new_uv = None;
         let mut next_orig_uv = Some(ty::UniverseIndex::ROOT);
         while let Some(orig_uv) = next_orig_uv.take() {
-            let mut update_uv = |var: &mut CanonicalVarInfo<I>, orig_uv, is_existential| {
+            let mut update_uv = |var: &mut CanonicalVarKind<I>, orig_uv, is_existential| {
                 let uv = var.universe();
                 match uv.cmp(&orig_uv) {
                     Ordering::Less => (), // Already updated
@@ -284,7 +317,7 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
             // Whenever we compress the universe of a placeholder, no existential with
             // an already compressed universe can name that placeholder.
             for is_existential in [false, true] {
-                for var in var_infos.iter_mut() {
+                for var in var_kinds.iter_mut() {
                     // We simply put all regions from the input into the highest
                     // compressed universe, so we only deal with them at the end.
                     if !var.is_region() {
@@ -298,26 +331,26 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
 
         // We put all regions into a separate universe.
         let mut first_region = true;
-        for var in var_infos.iter_mut() {
+        for var in var_kinds.iter_mut() {
             if var.is_region() {
                 if first_region {
                     first_region = false;
                     curr_compressed_uv = curr_compressed_uv.next_universe();
                 }
-                assert!(var.is_existential());
+                debug_assert!(var.is_existential());
                 *var = var.with_updated_universe(curr_compressed_uv);
             }
         }
 
-        let var_infos = self.delegate.cx().mk_canonical_var_infos(&var_infos);
-        (curr_compressed_uv, var_infos)
+        let var_kinds = self.delegate.cx().mk_canonical_var_kinds(&var_kinds);
+        (curr_compressed_uv, var_kinds)
     }
 
     fn cached_fold_ty(&mut self, t: I::Ty) -> I::Ty {
         let kind = match t.kind() {
             ty::Infer(i) => match i {
                 ty::TyVar(vid) => {
-                    assert_eq!(
+                    debug_assert_eq!(
                         self.delegate.opportunistic_resolve_ty_var(vid),
                         t,
                         "ty vid should have been resolved fully before canonicalization"
@@ -330,7 +363,7 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
                     ))
                 }
                 ty::IntVar(vid) => {
-                    assert_eq!(
+                    debug_assert_eq!(
                         self.delegate.opportunistic_resolve_int_var(vid),
                         t,
                         "ty vid should have been resolved fully before canonicalization"
@@ -338,7 +371,7 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
                     CanonicalVarKind::Ty(CanonicalTyVarKind::Int)
                 }
                 ty::FloatVar(vid) => {
-                    assert_eq!(
+                    debug_assert_eq!(
                         self.delegate.opportunistic_resolve_float_var(vid),
                         t,
                         "ty vid should have been resolved fully before canonicalization"
@@ -387,11 +420,15 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
             | ty::Alias(_, _)
             | ty::Bound(_, _)
             | ty::Error(_) => {
-                return ensure_sufficient_stack(|| t.super_fold_with(self));
+                return if t.has_type_flags(NEEDS_CANONICAL) {
+                    ensure_sufficient_stack(|| t.super_fold_with(self))
+                } else {
+                    t
+                };
             }
         };
 
-        let var = self.get_or_insert_bound_var(t, CanonicalVarInfo { kind });
+        let var = self.get_or_insert_bound_var(t, kind);
 
         Ty::new_anon_bound(self.cx(), self.binder_index, var)
     }
@@ -459,7 +496,7 @@ impl<D: SolverDelegate<Interner = I>, I: Interner> TypeFolder<I> for Canonicaliz
             },
 
             ty::ReVar(vid) => {
-                assert_eq!(
+                debug_assert_eq!(
                     self.delegate.opportunistic_resolve_lt_var(vid),
                     r,
                     "region vid should have been resolved fully before canonicalization"
@@ -475,7 +512,7 @@ impl<D: SolverDelegate<Interner = I>, I: Interner> TypeFolder<I> for Canonicaliz
             }
         };
 
-        let var = self.get_or_insert_bound_var(r, CanonicalVarInfo { kind });
+        let var = self.get_or_insert_bound_var(r, kind);
 
         Region::new_anon_bound(self.cx(), self.binder_index, var)
     }
@@ -485,7 +522,8 @@ impl<D: SolverDelegate<Interner = I>, I: Interner> TypeFolder<I> for Canonicaliz
             ty
         } else {
             let res = self.cached_fold_ty(t);
-            assert!(self.cache.insert((self.binder_index, t), res).is_none());
+            let old = self.cache.insert((self.binder_index, t), res);
+            assert_eq!(old, None);
             res
         }
     }
@@ -494,7 +532,7 @@ impl<D: SolverDelegate<Interner = I>, I: Interner> TypeFolder<I> for Canonicaliz
         let kind = match c.kind() {
             ty::ConstKind::Infer(i) => match i {
                 ty::InferConst::Var(vid) => {
-                    assert_eq!(
+                    debug_assert_eq!(
                         self.delegate.opportunistic_resolve_ct_var(vid),
                         c,
                         "const vid should have been resolved fully before canonicalization"
@@ -522,11 +560,28 @@ impl<D: SolverDelegate<Interner = I>, I: Interner> TypeFolder<I> for Canonicaliz
             | ty::ConstKind::Unevaluated(_)
             | ty::ConstKind::Value(_)
             | ty::ConstKind::Error(_)
-            | ty::ConstKind::Expr(_) => return c.super_fold_with(self),
+            | ty::ConstKind::Expr(_) => {
+                return if c.has_type_flags(NEEDS_CANONICAL) { c.super_fold_with(self) } else { c };
+            }
         };
 
-        let var = self.get_or_insert_bound_var(c, CanonicalVarInfo { kind });
+        let var = self.get_or_insert_bound_var(c, kind);
 
         Const::new_anon_bound(self.cx(), self.binder_index, var)
     }
+
+    fn fold_predicate(&mut self, p: I::Predicate) -> I::Predicate {
+        if p.flags().intersects(NEEDS_CANONICAL) { p.super_fold_with(self) } else { p }
+    }
+
+    fn fold_clauses(&mut self, c: I::Clauses) -> I::Clauses {
+        match self.canonicalize_mode {
+            CanonicalizeMode::Input { keep_static: true }
+            | CanonicalizeMode::Response { max_input_universe: _ } => {}
+            CanonicalizeMode::Input { keep_static: false } => {
+                panic!("erasing 'static in env")
+            }
+        }
+        if c.flags().intersects(NEEDS_CANONICAL) { c.super_fold_with(self) } else { c }
+    }
 }
diff --git a/compiler/rustc_next_trait_solver/src/delegate.rs b/compiler/rustc_next_trait_solver/src/delegate.rs
index 90a7c2e9f78..7b932010d49 100644
--- a/compiler/rustc_next_trait_solver/src/delegate.rs
+++ b/compiler/rustc_next_trait_solver/src/delegate.rs
@@ -17,6 +17,12 @@ pub trait SolverDelegate: Deref<Target = Self::Infcx> + Sized {
     where
         V: TypeFoldable<Self::Interner>;
 
+    fn compute_goal_fast_path(
+        &self,
+        goal: Goal<Self::Interner, <Self::Interner as Interner>::Predicate>,
+        span: <Self::Interner as Interner>::Span,
+    ) -> Option<Certainty>;
+
     fn fresh_var_for_kind_with_span(
         &self,
         arg: <Self::Interner as Interner>::GenericArg,
@@ -53,7 +59,7 @@ pub trait SolverDelegate: Deref<Target = Self::Infcx> + Sized {
 
     fn instantiate_canonical_var_with_infer(
         &self,
-        cv_info: ty::CanonicalVarInfo<Self::Interner>,
+        kind: ty::CanonicalVarKind<Self::Interner>,
         span: <Self::Interner as Interner>::Span,
         universe_map: impl Fn(ty::UniverseIndex) -> ty::UniverseIndex,
     ) -> <Self::Interner as Interner>::GenericArg;
diff --git a/compiler/rustc_next_trait_solver/src/resolve.rs b/compiler/rustc_next_trait_solver/src/resolve.rs
index 992c5ddf504..c3c57eccd6e 100644
--- a/compiler/rustc_next_trait_solver/src/resolve.rs
+++ b/compiler/rustc_next_trait_solver/src/resolve.rs
@@ -11,7 +11,7 @@ use crate::delegate::SolverDelegate;
 // EAGER RESOLUTION
 
 /// Resolves ty, region, and const vars to their inferred values or their root vars.
-pub struct EagerResolver<'a, D, I = <D as SolverDelegate>::Interner>
+struct EagerResolver<'a, D, I = <D as SolverDelegate>::Interner>
 where
     D: SolverDelegate<Interner = I>,
     I: Interner,
@@ -22,8 +22,20 @@ where
     cache: DelayedMap<I::Ty, I::Ty>,
 }
 
+pub fn eager_resolve_vars<D: SolverDelegate, T: TypeFoldable<D::Interner>>(
+    delegate: &D,
+    value: T,
+) -> T {
+    if value.has_infer() {
+        let mut folder = EagerResolver::new(delegate);
+        value.fold_with(&mut folder)
+    } else {
+        value
+    }
+}
+
 impl<'a, D: SolverDelegate> EagerResolver<'a, D> {
-    pub fn new(delegate: &'a D) -> Self {
+    fn new(delegate: &'a D) -> Self {
         EagerResolver { delegate, cache: Default::default() }
     }
 }
@@ -86,4 +98,12 @@ impl<D: SolverDelegate<Interner = I>, I: Interner> TypeFolder<I> for EagerResolv
             }
         }
     }
+
+    fn fold_predicate(&mut self, p: I::Predicate) -> I::Predicate {
+        if p.has_infer() { p.super_fold_with(self) } else { p }
+    }
+
+    fn fold_clauses(&mut self, c: I::Clauses) -> I::Clauses {
+        if c.has_infer() { c.super_fold_with(self) } else { c }
+    }
 }
diff --git a/compiler/rustc_next_trait_solver/src/solve/assembly/structural_traits.rs b/compiler/rustc_next_trait_solver/src/solve/assembly/structural_traits.rs
index 2a2b462a36c..4a2a7761f7f 100644
--- a/compiler/rustc_next_trait_solver/src/solve/assembly/structural_traits.rs
+++ b/compiler/rustc_next_trait_solver/src/solve/assembly/structural_traits.rs
@@ -83,7 +83,7 @@ where
             .cx()
             .coroutine_hidden_types(def_id)
             .instantiate(cx, args)
-            .map_bound(|tys| tys.to_vec())),
+            .map_bound(|bound| bound.types.to_vec())),
 
         ty::UnsafeBinder(bound_ty) => Ok(bound_ty.map_bound(|ty| vec![ty])),
 
@@ -249,7 +249,7 @@ where
             .cx()
             .coroutine_hidden_types(def_id)
             .instantiate(ecx.cx(), args)
-            .map_bound(|tys| tys.to_vec())),
+            .map_bound(|bound| bound.types.to_vec())),
     }
 }
 
@@ -327,7 +327,7 @@ pub(in crate::solve) fn extract_tupled_inputs_and_output_from_callable<I: Intern
                 // always be called once. It additionally implements `Fn`/`FnMut`
                 // only if it has no upvars referencing the closure-env lifetime,
                 // and if the closure kind permits it.
-                if closure_kind != ty::ClosureKind::FnOnce && args.has_self_borrows() {
+                if goal_kind != ty::ClosureKind::FnOnce && args.has_self_borrows() {
                     return Err(NoSolution);
                 }
 
diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/canonical.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/canonical.rs
index 36f68808a2c..cb7c498b94e 100644
--- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/canonical.rs
+++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/canonical.rs
@@ -12,6 +12,7 @@
 use std::iter;
 
 use rustc_index::IndexVec;
+use rustc_type_ir::data_structures::HashSet;
 use rustc_type_ir::inherent::*;
 use rustc_type_ir::relate::solver_relating::RelateExt;
 use rustc_type_ir::{
@@ -21,7 +22,7 @@ use tracing::{debug, instrument, trace};
 
 use crate::canonicalizer::Canonicalizer;
 use crate::delegate::SolverDelegate;
-use crate::resolve::EagerResolver;
+use crate::resolve::eager_resolve_vars;
 use crate::solve::eval_ctxt::CurrentGoalKind;
 use crate::solve::{
     CanonicalInput, CanonicalResponse, Certainty, EvalCtxt, ExternalConstraintsData, Goal,
@@ -52,16 +53,15 @@ where
 {
     /// Canonicalizes the goal remembering the original values
     /// for each bound variable.
-    pub(super) fn canonicalize_goal<T: TypeFoldable<I>>(
+    pub(super) fn canonicalize_goal(
         &self,
-        goal: Goal<I, T>,
-    ) -> (Vec<I::GenericArg>, CanonicalInput<I, T>) {
+        goal: Goal<I, I::Predicate>,
+    ) -> (Vec<I::GenericArg>, CanonicalInput<I, I::Predicate>) {
         // We only care about one entry per `OpaqueTypeKey` here,
         // so we only canonicalize the lookup table and ignore
         // duplicate entries.
         let opaque_types = self.delegate.clone_opaque_types_lookup_table();
-        let (goal, opaque_types) =
-            (goal, opaque_types).fold_with(&mut EagerResolver::new(self.delegate));
+        let (goal, opaque_types) = eager_resolve_vars(self.delegate, (goal, opaque_types));
 
         let mut orig_values = Default::default();
         let canonical = Canonicalizer::canonicalize_input(
@@ -129,7 +129,12 @@ where
                     if goals.is_empty() {
                         assert!(matches!(goals_certainty, Certainty::Yes));
                     }
-                    (Certainty::Yes, NestedNormalizationGoals(goals))
+                    (
+                        Certainty::Yes,
+                        NestedNormalizationGoals(
+                            goals.into_iter().map(|(s, g, _)| (s, g)).collect(),
+                        ),
+                    )
                 }
                 _ => {
                     let certainty = shallow_certainty.and(goals_certainty);
@@ -156,12 +161,14 @@ where
 
         let external_constraints =
             self.compute_external_query_constraints(certainty, normalization_nested_goals);
-        let (var_values, mut external_constraints) = (self.var_values, external_constraints)
-            .fold_with(&mut EagerResolver::new(self.delegate));
-        // Remove any trivial region constraints once we've resolved regions
-        external_constraints
-            .region_constraints
-            .retain(|outlives| outlives.0.as_region().is_none_or(|re| re != outlives.1));
+        let (var_values, mut external_constraints) =
+            eager_resolve_vars(self.delegate, (self.var_values, external_constraints));
+
+        // Remove any trivial or duplicated region constraints once we've resolved regions
+        let mut unique = HashSet::default();
+        external_constraints.region_constraints.retain(|outlives| {
+            outlives.0.as_region().is_none_or(|re| re != outlives.1) && unique.insert(*outlives)
+        });
 
         let canonical = Canonicalizer::canonicalize_response(
             self.delegate,
@@ -269,7 +276,7 @@ where
     pub(super) fn instantiate_and_apply_query_response(
         &mut self,
         param_env: I::ParamEnv,
-        original_values: Vec<I::GenericArg>,
+        original_values: &[I::GenericArg],
         response: CanonicalResponse<I>,
     ) -> (NestedNormalizationGoals<I>, Certainty) {
         let instantiation = Self::compute_query_response_instantiation_values(
@@ -357,15 +364,15 @@ where
         }
 
         let var_values = delegate.cx().mk_args_from_iter(
-            response.variables.iter().enumerate().map(|(index, info)| {
-                if info.universe() != ty::UniverseIndex::ROOT {
+            response.variables.iter().enumerate().map(|(index, var_kind)| {
+                if var_kind.universe() != ty::UniverseIndex::ROOT {
                     // A variable from inside a binder of the query. While ideally these shouldn't
                     // exist at all (see the FIXME at the start of this method), we have to deal with
                     // them for now.
-                    delegate.instantiate_canonical_var_with_infer(info, span, |idx| {
+                    delegate.instantiate_canonical_var_with_infer(var_kind, span, |idx| {
                         prev_universe + idx.index()
                     })
-                } else if info.is_existential() {
+                } else if var_kind.is_existential() {
                     // As an optimization we sometimes avoid creating a new inference variable here.
                     //
                     // All new inference variables we create start out in the current universe of the caller.
@@ -376,12 +383,13 @@ where
                     if let Some(v) = opt_values[ty::BoundVar::from_usize(index)] {
                         v
                     } else {
-                        delegate.instantiate_canonical_var_with_infer(info, span, |_| prev_universe)
+                        delegate
+                            .instantiate_canonical_var_with_infer(var_kind, span, |_| prev_universe)
                     }
                 } else {
                     // For placeholders which were already part of the input, we simply map this
                     // universal bound variable back the placeholder of the input.
-                    original_values[info.expect_placeholder_index()]
+                    original_values[var_kind.expect_placeholder_index()]
                 }
             }),
         );
@@ -465,7 +473,7 @@ where
 {
     let var_values = CanonicalVarValues { var_values: delegate.cx().mk_args(var_values) };
     let state = inspect::State { var_values, data };
-    let state = state.fold_with(&mut EagerResolver::new(delegate));
+    let state = eager_resolve_vars(delegate, state);
     Canonicalizer::canonicalize_response(delegate, max_input_universe, &mut vec![], state)
 }
 
diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs
index fc5dad9a3ed..345ece20b7e 100644
--- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs
+++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs
@@ -22,8 +22,9 @@ use crate::delegate::SolverDelegate;
 use crate::solve::inspect::{self, ProofTreeBuilder};
 use crate::solve::search_graph::SearchGraph;
 use crate::solve::{
-    CanonicalInput, Certainty, FIXPOINT_STEP_LIMIT, Goal, GoalEvaluationKind, GoalSource,
-    HasChanged, NestedNormalizationGoals, NoSolution, QueryInput, QueryResult,
+    CanonicalInput, Certainty, FIXPOINT_STEP_LIMIT, Goal, GoalEvaluation, GoalEvaluationKind,
+    GoalSource, GoalStalledOn, HasChanged, NestedNormalizationGoals, NoSolution, QueryInput,
+    QueryResult,
 };
 
 pub(super) mod canonical;
@@ -91,7 +92,7 @@ where
 
     /// The variable info for the `var_values`, only used to make an ambiguous response
     /// with no constraints.
-    variables: I::CanonicalVars,
+    variables: I::CanonicalVarKinds,
 
     /// What kind of goal we're currently computing, see the enum definition
     /// for more info.
@@ -115,7 +116,7 @@ where
 
     pub(super) search_graph: &'a mut SearchGraph<D>,
 
-    nested_goals: Vec<(GoalSource, Goal<I, I::Predicate>)>,
+    nested_goals: Vec<(GoalSource, Goal<I, I::Predicate>, Option<GoalStalledOn<I>>)>,
 
     pub(super) origin_span: I::Span,
 
@@ -147,8 +148,9 @@ pub trait SolverDelegateEvalExt: SolverDelegate {
         goal: Goal<Self::Interner, <Self::Interner as Interner>::Predicate>,
         generate_proof_tree: GenerateProofTree,
         span: <Self::Interner as Interner>::Span,
+        stalled_on: Option<GoalStalledOn<Self::Interner>>,
     ) -> (
-        Result<(HasChanged, Certainty), NoSolution>,
+        Result<GoalEvaluation<Self::Interner>, NoSolution>,
         Option<inspect::GoalEvaluation<Self::Interner>>,
     );
 
@@ -171,8 +173,12 @@ pub trait SolverDelegateEvalExt: SolverDelegate {
         &self,
         goal: Goal<Self::Interner, <Self::Interner as Interner>::Predicate>,
         generate_proof_tree: GenerateProofTree,
+        stalled_on: Option<GoalStalledOn<Self::Interner>>,
     ) -> (
-        Result<(NestedNormalizationGoals<Self::Interner>, HasChanged, Certainty), NoSolution>,
+        Result<
+            (NestedNormalizationGoals<Self::Interner>, GoalEvaluation<Self::Interner>),
+            NoSolution,
+        >,
         Option<inspect::GoalEvaluation<Self::Interner>>,
     );
 }
@@ -188,9 +194,10 @@ where
         goal: Goal<I, I::Predicate>,
         generate_proof_tree: GenerateProofTree,
         span: I::Span,
-    ) -> (Result<(HasChanged, Certainty), NoSolution>, Option<inspect::GoalEvaluation<I>>) {
+        stalled_on: Option<GoalStalledOn<I>>,
+    ) -> (Result<GoalEvaluation<I>, NoSolution>, Option<inspect::GoalEvaluation<I>>) {
         EvalCtxt::enter_root(self, self.cx().recursion_limit(), generate_proof_tree, span, |ecx| {
-            ecx.evaluate_goal(GoalEvaluationKind::Root, GoalSource::Misc, goal)
+            ecx.evaluate_goal(GoalEvaluationKind::Root, GoalSource::Misc, goal, stalled_on)
         })
     }
 
@@ -201,7 +208,7 @@ where
     ) -> bool {
         self.probe(|| {
             EvalCtxt::enter_root(self, root_depth, GenerateProofTree::No, I::Span::dummy(), |ecx| {
-                ecx.evaluate_goal(GoalEvaluationKind::Root, GoalSource::Misc, goal)
+                ecx.evaluate_goal(GoalEvaluationKind::Root, GoalSource::Misc, goal, None)
             })
             .0
         })
@@ -213,8 +220,9 @@ where
         &self,
         goal: Goal<I, I::Predicate>,
         generate_proof_tree: GenerateProofTree,
+        stalled_on: Option<GoalStalledOn<I>>,
     ) -> (
-        Result<(NestedNormalizationGoals<I>, HasChanged, Certainty), NoSolution>,
+        Result<(NestedNormalizationGoals<I>, GoalEvaluation<I>), NoSolution>,
         Option<inspect::GoalEvaluation<I>>,
     ) {
         EvalCtxt::enter_root(
@@ -222,7 +230,9 @@ where
             self.cx().recursion_limit(),
             generate_proof_tree,
             I::Span::dummy(),
-            |ecx| ecx.evaluate_goal_raw(GoalEvaluationKind::Root, GoalSource::Misc, goal),
+            |ecx| {
+                ecx.evaluate_goal_raw(GoalEvaluationKind::Root, GoalSource::Misc, goal, stalled_on)
+            },
         )
     }
 }
@@ -447,11 +457,12 @@ where
         goal_evaluation_kind: GoalEvaluationKind,
         source: GoalSource,
         goal: Goal<I, I::Predicate>,
-    ) -> Result<(HasChanged, Certainty), NoSolution> {
-        let (normalization_nested_goals, has_changed, certainty) =
-            self.evaluate_goal_raw(goal_evaluation_kind, source, goal)?;
+        stalled_on: Option<GoalStalledOn<I>>,
+    ) -> Result<GoalEvaluation<I>, NoSolution> {
+        let (normalization_nested_goals, goal_evaluation) =
+            self.evaluate_goal_raw(goal_evaluation_kind, source, goal, stalled_on)?;
         assert!(normalization_nested_goals.is_empty());
-        Ok((has_changed, certainty))
+        Ok(goal_evaluation)
     }
 
     /// Recursively evaluates `goal`, returning the nested goals in case
@@ -466,7 +477,29 @@ where
         goal_evaluation_kind: GoalEvaluationKind,
         source: GoalSource,
         goal: Goal<I, I::Predicate>,
-    ) -> Result<(NestedNormalizationGoals<I>, HasChanged, Certainty), NoSolution> {
+        stalled_on: Option<GoalStalledOn<I>>,
+    ) -> Result<(NestedNormalizationGoals<I>, GoalEvaluation<I>), NoSolution> {
+        // If we have run this goal before, and it was stalled, check that any of the goal's
+        // args have changed. Otherwise, we don't need to re-run the goal because it'll remain
+        // stalled, since it'll canonicalize the same way and evaluation is pure.
+        if let Some(stalled_on) = stalled_on {
+            if !stalled_on.stalled_vars.iter().any(|value| self.delegate.is_changed_arg(*value))
+                && !self
+                    .delegate
+                    .opaque_types_storage_num_entries()
+                    .needs_reevaluation(stalled_on.num_opaques)
+            {
+                return Ok((
+                    NestedNormalizationGoals::empty(),
+                    GoalEvaluation {
+                        certainty: Certainty::Maybe(stalled_on.stalled_cause),
+                        has_changed: HasChanged::No,
+                        stalled_on: Some(stalled_on),
+                    },
+                ));
+            }
+        }
+
         let (orig_values, canonical_goal) = self.canonicalize_goal(goal);
         let mut goal_evaluation =
             self.inspect.new_goal_evaluation(goal, &orig_values, goal_evaluation_kind);
@@ -489,7 +522,7 @@ where
             if !has_only_region_constraints(response) { HasChanged::Yes } else { HasChanged::No };
 
         let (normalization_nested_goals, certainty) =
-            self.instantiate_and_apply_query_response(goal.param_env, orig_values, response);
+            self.instantiate_and_apply_query_response(goal.param_env, &orig_values, response);
         self.inspect.goal_evaluation(goal_evaluation);
 
         // FIXME: We previously had an assert here that checked that recomputing
@@ -502,7 +535,53 @@ where
         // Once we have decided on how to handle trait-system-refactor-initiative#75,
         // we should re-add an assert here.
 
-        Ok((normalization_nested_goals, has_changed, certainty))
+        let stalled_on = match certainty {
+            Certainty::Yes => None,
+            Certainty::Maybe(stalled_cause) => match has_changed {
+                // FIXME: We could recompute a *new* set of stalled variables by walking
+                // through the orig values, resolving, and computing the root vars of anything
+                // that is not resolved. Only when *these* have changed is it meaningful
+                // to recompute this goal.
+                HasChanged::Yes => None,
+                HasChanged::No => {
+                    let mut stalled_vars = orig_values;
+
+                    // Remove the canonicalized universal vars, since we only care about stalled existentials.
+                    stalled_vars.retain(|arg| match arg.kind() {
+                        ty::GenericArgKind::Type(ty) => matches!(ty.kind(), ty::Infer(_)),
+                        ty::GenericArgKind::Const(ct) => {
+                            matches!(ct.kind(), ty::ConstKind::Infer(_))
+                        }
+                        // Lifetimes can never stall goals.
+                        ty::GenericArgKind::Lifetime(_) => false,
+                    });
+
+                    // Remove the unconstrained RHS arg, which is expected to have changed.
+                    if let Some(normalizes_to) = goal.predicate.as_normalizes_to() {
+                        let normalizes_to = normalizes_to.skip_binder();
+                        let rhs_arg: I::GenericArg = normalizes_to.term.into();
+                        let idx = stalled_vars
+                            .iter()
+                            .rposition(|arg| *arg == rhs_arg)
+                            .expect("expected unconstrained arg");
+                        stalled_vars.swap_remove(idx);
+                    }
+
+                    Some(GoalStalledOn {
+                        num_opaques: canonical_goal
+                            .canonical
+                            .value
+                            .predefined_opaques_in_body
+                            .opaque_types
+                            .len(),
+                        stalled_vars,
+                        stalled_cause,
+                    })
+                }
+            },
+        };
+
+        Ok((normalization_nested_goals, GoalEvaluation { certainty, has_changed, stalled_on }))
     }
 
     fn compute_goal(&mut self, goal: Goal<I, I::Predicate>) -> QueryResult<I> {
@@ -602,7 +681,18 @@ where
         let cx = self.cx();
         // If this loop did not result in any progress, what's our final certainty.
         let mut unchanged_certainty = Some(Certainty::Yes);
-        for (source, goal) in mem::take(&mut self.nested_goals) {
+        for (source, goal, stalled_on) in mem::take(&mut self.nested_goals) {
+            if let Some(certainty) = self.delegate.compute_goal_fast_path(goal, self.origin_span) {
+                match certainty {
+                    Certainty::Yes => {}
+                    Certainty::Maybe(_) => {
+                        self.nested_goals.push((source, goal, None));
+                        unchanged_certainty = unchanged_certainty.map(|c| c.and(certainty));
+                    }
+                }
+                continue;
+            }
+
             // We treat normalizes-to goals specially here. In each iteration we take the
             // RHS of the projection, replace it with a fresh inference variable, and only
             // after evaluating that goal do we equate the fresh inference variable with the
@@ -622,11 +712,18 @@ where
                 let unconstrained_goal =
                     goal.with(cx, ty::NormalizesTo { alias: pred.alias, term: unconstrained_rhs });
 
-                let (NestedNormalizationGoals(nested_goals), _, certainty) =
-                    self.evaluate_goal_raw(GoalEvaluationKind::Nested, source, unconstrained_goal)?;
+                let (
+                    NestedNormalizationGoals(nested_goals),
+                    GoalEvaluation { certainty, stalled_on, has_changed: _ },
+                ) = self.evaluate_goal_raw(
+                    GoalEvaluationKind::Nested,
+                    source,
+                    unconstrained_goal,
+                    stalled_on,
+                )?;
                 // Add the nested goals from normalization to our own nested goals.
                 trace!(?nested_goals);
-                self.nested_goals.extend(nested_goals);
+                self.nested_goals.extend(nested_goals.into_iter().map(|(s, g)| (s, g, None)));
 
                 // Finally, equate the goal's RHS with the unconstrained var.
                 //
@@ -652,6 +749,8 @@ where
                 // looking at the "has changed" return from evaluate_goal,
                 // because we expect the `unconstrained_rhs` part of the predicate
                 // to have changed -- that means we actually normalized successfully!
+                // FIXME: Do we need to eagerly resolve here? Or should we check
+                // if the cache key has any changed vars?
                 let with_resolved_vars = self.resolve_vars_if_possible(goal);
                 if pred.alias != goal.predicate.as_normalizes_to().unwrap().skip_binder().alias {
                     unchanged_certainty = None;
@@ -660,13 +759,13 @@ where
                 match certainty {
                     Certainty::Yes => {}
                     Certainty::Maybe(_) => {
-                        self.nested_goals.push((source, with_resolved_vars));
+                        self.nested_goals.push((source, with_resolved_vars, stalled_on));
                         unchanged_certainty = unchanged_certainty.map(|c| c.and(certainty));
                     }
                 }
             } else {
-                let (has_changed, certainty) =
-                    self.evaluate_goal(GoalEvaluationKind::Nested, source, goal)?;
+                let GoalEvaluation { certainty, has_changed, stalled_on } =
+                    self.evaluate_goal(GoalEvaluationKind::Nested, source, goal, stalled_on)?;
                 if has_changed == HasChanged::Yes {
                     unchanged_certainty = None;
                 }
@@ -674,7 +773,7 @@ where
                 match certainty {
                     Certainty::Yes => {}
                     Certainty::Maybe(_) => {
-                        self.nested_goals.push((source, goal));
+                        self.nested_goals.push((source, goal, stalled_on));
                         unchanged_certainty = unchanged_certainty.map(|c| c.and(certainty));
                     }
                 }
@@ -698,7 +797,7 @@ where
         goal.predicate =
             goal.predicate.fold_with(&mut ReplaceAliasWithInfer::new(self, source, goal.param_env));
         self.inspect.add_goal(self.delegate, self.max_input_universe, source, goal);
-        self.nested_goals.push((source, goal));
+        self.nested_goals.push((source, goal, None));
     }
 
     #[instrument(level = "trace", skip(self, goals))]
@@ -732,8 +831,8 @@ where
 
     /// Returns a ty infer or a const infer depending on whether `kind` is a `Ty` or `Const`.
     /// If `kind` is an integer inference variable this will still return a ty infer var.
-    pub(super) fn next_term_infer_of_kind(&mut self, kind: I::Term) -> I::Term {
-        match kind.kind() {
+    pub(super) fn next_term_infer_of_kind(&mut self, term: I::Term) -> I::Term {
+        match term.kind() {
             ty::TermKind::Ty(_) => self.next_ty_infer().into(),
             ty::TermKind::Const(_) => self.next_const_infer().into(),
         }
@@ -840,6 +939,22 @@ where
                     }
                 }
             }
+
+            fn visit_predicate(&mut self, p: I::Predicate) -> Self::Result {
+                if p.has_non_region_infer() || p.has_placeholders() {
+                    p.super_visit_with(self)
+                } else {
+                    ControlFlow::Continue(())
+                }
+            }
+
+            fn visit_clauses(&mut self, c: I::Clauses) -> Self::Result {
+                if c.has_non_region_infer() || c.has_placeholders() {
+                    c.super_visit_with(self)
+                } else {
+                    ControlFlow::Continue(())
+                }
+            }
         }
 
         let mut visitor = ContainsTermOrNotNameable {
@@ -1016,7 +1131,7 @@ where
     }
 
     pub(super) fn register_region_outlives(&self, a: I::Region, b: I::Region) {
-        // `b : a` ==> `a <= b`
+        // `'a: 'b` ==> `'b <= 'a`
         self.delegate.sub_regions(b, a, self.origin_span);
     }
 
diff --git a/compiler/rustc_next_trait_solver/src/solve/mod.rs b/compiler/rustc_next_trait_solver/src/solve/mod.rs
index 8173146e2fe..a7a984181d7 100644
--- a/compiler/rustc_next_trait_solver/src/solve/mod.rs
+++ b/compiler/rustc_next_trait_solver/src/solve/mod.rs
@@ -21,6 +21,7 @@ mod project_goals;
 mod search_graph;
 mod trait_goals;
 
+use derive_where::derive_where;
 use rustc_type_ir::inherent::*;
 pub use rustc_type_ir::solve::*;
 use rustc_type_ir::{self as ty, Interner, TypingMode};
@@ -354,7 +355,7 @@ where
 fn response_no_constraints_raw<I: Interner>(
     cx: I,
     max_universe: ty::UniverseIndex,
-    variables: I::CanonicalVars,
+    variables: I::CanonicalVarKinds,
     certainty: Certainty,
 ) -> CanonicalResponse<I> {
     ty::Canonical {
@@ -369,3 +370,21 @@ fn response_no_constraints_raw<I: Interner>(
         },
     }
 }
+
+/// The result of evaluating a goal.
+pub struct GoalEvaluation<I: Interner> {
+    pub certainty: Certainty,
+    pub has_changed: HasChanged,
+    /// If the [`Certainty`] was `Maybe`, then keep track of whether the goal has changed
+    /// before rerunning it.
+    pub stalled_on: Option<GoalStalledOn<I>>,
+}
+
+/// The conditions that must change for a goal to warrant
+#[derive_where(Clone, Debug; I: Interner)]
+pub struct GoalStalledOn<I: Interner> {
+    pub num_opaques: usize,
+    pub stalled_vars: Vec<I::GenericArg>,
+    /// The cause that will be returned on subsequent evaluations if this goal remains stalled.
+    pub stalled_cause: MaybeCause,
+}
diff --git a/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs b/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs
index 966d5422fbb..7c4e1dc2c12 100644
--- a/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs
+++ b/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs
@@ -9,7 +9,7 @@ use rustc_type_ir::{
     self as ty, Interner, Movability, TraitPredicate, TypeVisitableExt as _, TypingMode,
     Upcast as _, elaborate,
 };
-use tracing::{instrument, trace};
+use tracing::{debug, instrument, trace};
 
 use crate::delegate::SolverDelegate;
 use crate::solve::assembly::structural_traits::{self, AsyncCallableRelevantTypes};
@@ -17,7 +17,7 @@ use crate::solve::assembly::{self, AllowInferenceConstraints, AssembleCandidates
 use crate::solve::inspect::ProbeKind;
 use crate::solve::{
     BuiltinImplSource, CandidateSource, Certainty, EvalCtxt, Goal, GoalSource, MaybeCause,
-    NoSolution, ParamEnvSource, QueryResult,
+    NoSolution, ParamEnvSource, QueryResult, has_only_region_constraints,
 };
 
 impl<D, I> assembly::GoalKind<D> for TraitPredicate<I>
@@ -1253,6 +1253,45 @@ where
     D: SolverDelegate<Interner = I>,
     I: Interner,
 {
+    /// FIXME(#57893): For backwards compatability with the old trait solver implementation,
+    /// we need to handle overlap between builtin and user-written impls for trait objects.
+    ///
+    /// This overlap is unsound in general and something which we intend to fix separately.
+    /// To avoid blocking the stabilization of the trait solver, we add this hack to avoid
+    /// breakage in cases which are *mostly fine*™. Importantly, this preference is strictly
+    /// weaker than the old behavior.
+    ///
+    /// We only prefer builtin over user-written impls if there are no inference constraints.
+    /// Importantly, we also only prefer the builtin impls for trait goals, and not during
+    /// normalization. This means the only case where this special-case results in exploitable
+    /// unsoundness should be lifetime dependent user-written impls.
+    pub(super) fn unsound_prefer_builtin_dyn_impl(&mut self, candidates: &mut Vec<Candidate<I>>) {
+        match self.typing_mode() {
+            TypingMode::Coherence => return,
+            TypingMode::Analysis { .. }
+            | TypingMode::Borrowck { .. }
+            | TypingMode::PostBorrowckAnalysis { .. }
+            | TypingMode::PostAnalysis => {}
+        }
+
+        if candidates
+            .iter()
+            .find(|c| {
+                matches!(c.source, CandidateSource::BuiltinImpl(BuiltinImplSource::Object(_)))
+            })
+            .is_some_and(|c| has_only_region_constraints(c.result))
+        {
+            candidates.retain(|c| {
+                if matches!(c.source, CandidateSource::Impl(_)) {
+                    debug!(?c, "unsoundly dropping impl in favor of builtin dyn-candidate");
+                    false
+                } else {
+                    true
+                }
+            });
+        }
+    }
+
     #[instrument(level = "debug", skip(self), ret)]
     pub(super) fn merge_trait_candidates(
         &mut self,
@@ -1313,6 +1352,7 @@ where
         }
 
         self.filter_specialized_impls(AllowInferenceConstraints::No, &mut candidates);
+        self.unsound_prefer_builtin_dyn_impl(&mut candidates);
 
         // If there are *only* global where bounds, then make sure to return that this
         // is still reported as being proven-via the param-env so that rigid projections
diff --git a/compiler/rustc_parse/messages.ftl b/compiler/rustc_parse/messages.ftl
index a6919afef12..1f221b4bf78 100644
--- a/compiler/rustc_parse/messages.ftl
+++ b/compiler/rustc_parse/messages.ftl
@@ -8,6 +8,30 @@ parse_array_brackets_instead_of_braces = this is a block expression, not an arra
 
 parse_array_index_offset_of = array indexing not supported in offset_of
 
+parse_asm_expected_comma = expected token: `,`
+    .label = expected `,`
+
+parse_asm_expected_other = expected operand, {$is_inline_asm ->
+    [false] options
+    *[true] clobber_abi, options
+    }, or additional template string
+
+parse_asm_expected_register_class_or_explicit_register = expected register class or explicit register
+
+parse_asm_expected_string_literal = expected string literal
+    .label = not a string literal
+
+parse_asm_non_abi = at least one abi must be provided as an argument to `clobber_abi`
+
+parse_asm_requires_template = requires at least a template string argument
+
+parse_asm_sym_no_path = expected a path for argument to `sym`
+
+parse_asm_underscore_input = _ cannot be used for input operands
+
+parse_asm_unsupported_operand = the `{$symbol}` operand cannot be used with `{$macro_name}!`
+    .label = the `{$symbol}` operand is not meaningful for global-scoped inline assembly, remove it
+
 parse_assignment_else_not_allowed = <assignment> ... else {"{"} ... {"}"} is not allowed
 
 parse_associated_static_item_not_allowed = associated `static` items are not allowed
diff --git a/compiler/rustc_parse/src/errors.rs b/compiler/rustc_parse/src/errors.rs
index 31a48b22cfe..2dba568a258 100644
--- a/compiler/rustc_parse/src/errors.rs
+++ b/compiler/rustc_parse/src/errors.rs
@@ -3525,3 +3525,73 @@ pub(crate) struct MoveSelfModifier {
     pub insertion_span: Span,
     pub modifier: String,
 }
+
+#[derive(Diagnostic)]
+#[diag(parse_asm_unsupported_operand)]
+pub(crate) struct AsmUnsupportedOperand<'a> {
+    #[primary_span]
+    #[label]
+    pub(crate) span: Span,
+    pub(crate) symbol: &'a str,
+    pub(crate) macro_name: &'static str,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_asm_underscore_input)]
+pub(crate) struct AsmUnderscoreInput {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_asm_sym_no_path)]
+pub(crate) struct AsmSymNoPath {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_asm_requires_template)]
+pub(crate) struct AsmRequiresTemplate {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_asm_expected_comma)]
+pub(crate) struct AsmExpectedComma {
+    #[primary_span]
+    #[label]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_asm_expected_other)]
+pub(crate) struct AsmExpectedOther {
+    #[primary_span]
+    #[label(parse_asm_expected_other)]
+    pub(crate) span: Span,
+    pub(crate) is_inline_asm: bool,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_asm_non_abi)]
+pub(crate) struct NonABI {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_asm_expected_string_literal)]
+pub(crate) struct AsmExpectedStringLiteral {
+    #[primary_span]
+    #[label]
+    pub(crate) span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_asm_expected_register_class_or_explicit_register)]
+pub(crate) struct ExpectedRegisterClassOrExplicitRegister {
+    #[primary_span]
+    pub(crate) span: Span,
+}
diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs
index 78c5742414b..2845bbed1c0 100644
--- a/compiler/rustc_parse/src/lexer/mod.rs
+++ b/compiler/rustc_parse/src/lexer/mod.rs
@@ -4,7 +4,7 @@ use diagnostics::make_unclosed_delims_error;
 use rustc_ast::ast::{self, AttrStyle};
 use rustc_ast::token::{self, CommentKind, Delimiter, IdentIsRaw, Token, TokenKind};
 use rustc_ast::tokenstream::TokenStream;
-use rustc_ast::util::unicode::contains_text_flow_control_chars;
+use rustc_ast::util::unicode::{TEXT_FLOW_CONTROL_CHARS, contains_text_flow_control_chars};
 use rustc_errors::codes::*;
 use rustc_errors::{Applicability, Diag, DiagCtxtHandle, StashKey};
 use rustc_lexer::{
@@ -14,7 +14,7 @@ use rustc_literal_escaper::{EscapeError, Mode, unescape_mixed, unescape_unicode}
 use rustc_session::lint::BuiltinLintDiag;
 use rustc_session::lint::builtin::{
     RUST_2021_PREFIXES_INCOMPATIBLE_SYNTAX, RUST_2024_GUARDED_STRING_INCOMPATIBLE_SYNTAX,
-    TEXT_DIRECTION_CODEPOINT_IN_COMMENT,
+    TEXT_DIRECTION_CODEPOINT_IN_COMMENT, TEXT_DIRECTION_CODEPOINT_IN_LITERAL,
 };
 use rustc_session::parse::ParseSess;
 use rustc_span::{BytePos, Pos, Span, Symbol, sym};
@@ -174,6 +174,7 @@ impl<'psess, 'src> Lexer<'psess, 'src> {
                     // Opening delimiter of the length 3 is not included into the symbol.
                     let content_start = start + BytePos(3);
                     let content = self.str_from(content_start);
+                    self.lint_doc_comment_unicode_text_flow(start, content);
                     self.cook_doc_comment(content_start, content, CommentKind::Line, doc_style)
                 }
                 rustc_lexer::TokenKind::BlockComment { doc_style, terminated } => {
@@ -193,6 +194,7 @@ impl<'psess, 'src> Lexer<'psess, 'src> {
                     let content_start = start + BytePos(3);
                     let content_end = self.pos - BytePos(if terminated { 2 } else { 0 });
                     let content = self.str_from_to(content_start, content_end);
+                    self.lint_doc_comment_unicode_text_flow(start, content);
                     self.cook_doc_comment(content_start, content, CommentKind::Block, doc_style)
                 }
                 rustc_lexer::TokenKind::Frontmatter { has_invalid_preceding_whitespace, invalid_infostring } => {
@@ -287,6 +289,7 @@ impl<'psess, 'src> Lexer<'psess, 'src> {
                     } else {
                         None
                     };
+                    self.lint_literal_unicode_text_flow(symbol, kind, self.mk_sp(start, self.pos), "literal");
                     token::Literal(token::Lit { kind, symbol, suffix })
                 }
                 rustc_lexer::TokenKind::Lifetime { starts_with_number } => {
@@ -481,6 +484,88 @@ impl<'psess, 'src> Lexer<'psess, 'src> {
         }
     }
 
+    fn lint_doc_comment_unicode_text_flow(&mut self, start: BytePos, content: &str) {
+        if contains_text_flow_control_chars(content) {
+            self.report_text_direction_codepoint(
+                content,
+                self.mk_sp(start, self.pos),
+                0,
+                false,
+                "doc comment",
+            );
+        }
+    }
+
+    fn lint_literal_unicode_text_flow(
+        &mut self,
+        text: Symbol,
+        lit_kind: token::LitKind,
+        span: Span,
+        label: &'static str,
+    ) {
+        if !contains_text_flow_control_chars(text.as_str()) {
+            return;
+        }
+        let (padding, point_at_inner_spans) = match lit_kind {
+            // account for `"` or `'`
+            token::LitKind::Str | token::LitKind::Char => (1, true),
+            // account for `c"`
+            token::LitKind::CStr => (2, true),
+            // account for `r###"`
+            token::LitKind::StrRaw(n) => (n as u32 + 2, true),
+            // account for `cr###"`
+            token::LitKind::CStrRaw(n) => (n as u32 + 3, true),
+            // suppress bad literals.
+            token::LitKind::Err(_) => return,
+            // Be conservative just in case new literals do support these.
+            _ => (0, false),
+        };
+        self.report_text_direction_codepoint(
+            text.as_str(),
+            span,
+            padding,
+            point_at_inner_spans,
+            label,
+        );
+    }
+
+    fn report_text_direction_codepoint(
+        &self,
+        text: &str,
+        span: Span,
+        padding: u32,
+        point_at_inner_spans: bool,
+        label: &str,
+    ) {
+        // Obtain the `Span`s for each of the forbidden chars.
+        let spans: Vec<_> = text
+            .char_indices()
+            .filter_map(|(i, c)| {
+                TEXT_FLOW_CONTROL_CHARS.contains(&c).then(|| {
+                    let lo = span.lo() + BytePos(i as u32 + padding);
+                    (c, span.with_lo(lo).with_hi(lo + BytePos(c.len_utf8() as u32)))
+                })
+            })
+            .collect();
+
+        let count = spans.len();
+        let labels = point_at_inner_spans.then_some(spans.clone());
+
+        self.psess.buffer_lint(
+            TEXT_DIRECTION_CODEPOINT_IN_LITERAL,
+            span,
+            ast::CRATE_NODE_ID,
+            BuiltinLintDiag::HiddenUnicodeCodepoints {
+                label: label.to_string(),
+                count,
+                span_label: span,
+                labels,
+                escape: point_at_inner_spans && !spans.is_empty(),
+                spans,
+            },
+        );
+    }
+
     fn validate_frontmatter(
         &self,
         start: BytePos,
diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs
index 3ab726d9d9d..8ea535599c9 100644
--- a/compiler/rustc_parse/src/lib.rs
+++ b/compiler/rustc_parse/src/lib.rs
@@ -9,7 +9,6 @@
 #![feature(debug_closure_helpers)]
 #![feature(if_let_guard)]
 #![feature(iter_intersperse)]
-#![feature(string_from_utf8_lossy_owned)]
 #![recursion_limit = "256"]
 // tidy-alphabetical-end
 
diff --git a/compiler/rustc_parse/src/parser/asm.rs b/compiler/rustc_parse/src/parser/asm.rs
new file mode 100644
index 00000000000..d4d0612a317
--- /dev/null
+++ b/compiler/rustc_parse/src/parser/asm.rs
@@ -0,0 +1,385 @@
+use rustc_ast::ptr::P;
+use rustc_ast::{self as ast, AsmMacro};
+use rustc_span::{Span, Symbol, kw};
+
+use super::{ExpKeywordPair, ForceCollect, IdentIsRaw, Trailing, UsePreAttrPos};
+use crate::{PResult, Parser, errors, exp, token};
+
+/// An argument to one of the `asm!` macros. The argument is syntactically valid, but is otherwise
+/// not validated at all.
+pub struct AsmArg {
+    pub kind: AsmArgKind,
+    pub attributes: AsmAttrVec,
+    pub span: Span,
+}
+
+pub enum AsmArgKind {
+    Template(P<ast::Expr>),
+    Operand(Option<Symbol>, ast::InlineAsmOperand),
+    Options(Vec<AsmOption>),
+    ClobberAbi(Vec<(Symbol, Span)>),
+}
+
+pub struct AsmOption {
+    pub symbol: Symbol,
+    pub span: Span,
+    // A bitset, with only the bit for this option's symbol set.
+    pub options: ast::InlineAsmOptions,
+    // Used when suggesting to remove an option.
+    pub span_with_comma: Span,
+}
+
+/// A parsed list of attributes that is not attached to any item.
+/// Used to check whether `asm!` arguments are configured out.
+pub struct AsmAttrVec(pub ast::AttrVec);
+
+impl AsmAttrVec {
+    fn parse<'a>(p: &mut Parser<'a>) -> PResult<'a, Self> {
+        let attrs = p.parse_outer_attributes()?;
+
+        p.collect_tokens(None, attrs, ForceCollect::No, |_, attrs| {
+            Ok((Self(attrs), Trailing::No, UsePreAttrPos::No))
+        })
+    }
+}
+impl ast::HasAttrs for AsmAttrVec {
+    // Follows `ast::Expr`.
+    const SUPPORTS_CUSTOM_INNER_ATTRS: bool = false;
+
+    fn attrs(&self) -> &[rustc_ast::Attribute] {
+        &self.0
+    }
+
+    fn visit_attrs(&mut self, f: impl FnOnce(&mut rustc_ast::AttrVec)) {
+        f(&mut self.0)
+    }
+}
+
+impl ast::HasTokens for AsmAttrVec {
+    fn tokens(&self) -> Option<&rustc_ast::tokenstream::LazyAttrTokenStream> {
+        None
+    }
+
+    fn tokens_mut(&mut self) -> Option<&mut Option<rustc_ast::tokenstream::LazyAttrTokenStream>> {
+        None
+    }
+}
+
+/// Used for better error messages when operand types are used that are not
+/// supported by the current macro (e.g. `in` or `out` for `global_asm!`)
+///
+/// returns
+///
+/// - `Ok(true)` if the current token matches the keyword, and was expected
+/// - `Ok(false)` if the current token does not match the keyword
+/// - `Err(_)` if the current token matches the keyword, but was not expected
+fn eat_operand_keyword<'a>(
+    p: &mut Parser<'a>,
+    exp: ExpKeywordPair,
+    asm_macro: AsmMacro,
+) -> PResult<'a, bool> {
+    if matches!(asm_macro, AsmMacro::Asm) {
+        Ok(p.eat_keyword(exp))
+    } else {
+        let span = p.token.span;
+        if p.eat_keyword_noexpect(exp.kw) {
+            // in gets printed as `r#in` otherwise
+            let symbol = if exp.kw == kw::In { "in" } else { exp.kw.as_str() };
+            Err(p.dcx().create_err(errors::AsmUnsupportedOperand {
+                span,
+                symbol,
+                macro_name: asm_macro.macro_name(),
+            }))
+        } else {
+            Ok(false)
+        }
+    }
+}
+
+fn parse_asm_operand<'a>(
+    p: &mut Parser<'a>,
+    asm_macro: AsmMacro,
+) -> PResult<'a, Option<ast::InlineAsmOperand>> {
+    let dcx = p.dcx();
+
+    Ok(Some(if eat_operand_keyword(p, exp!(In), asm_macro)? {
+        let reg = parse_reg(p)?;
+        if p.eat_keyword(exp!(Underscore)) {
+            let err = dcx.create_err(errors::AsmUnderscoreInput { span: p.token.span });
+            return Err(err);
+        }
+        let expr = p.parse_expr()?;
+        ast::InlineAsmOperand::In { reg, expr }
+    } else if eat_operand_keyword(p, exp!(Out), asm_macro)? {
+        let reg = parse_reg(p)?;
+        let expr = if p.eat_keyword(exp!(Underscore)) { None } else { Some(p.parse_expr()?) };
+        ast::InlineAsmOperand::Out { reg, expr, late: false }
+    } else if eat_operand_keyword(p, exp!(Lateout), asm_macro)? {
+        let reg = parse_reg(p)?;
+        let expr = if p.eat_keyword(exp!(Underscore)) { None } else { Some(p.parse_expr()?) };
+        ast::InlineAsmOperand::Out { reg, expr, late: true }
+    } else if eat_operand_keyword(p, exp!(Inout), asm_macro)? {
+        let reg = parse_reg(p)?;
+        if p.eat_keyword(exp!(Underscore)) {
+            let err = dcx.create_err(errors::AsmUnderscoreInput { span: p.token.span });
+            return Err(err);
+        }
+        let expr = p.parse_expr()?;
+        if p.eat(exp!(FatArrow)) {
+            let out_expr =
+                if p.eat_keyword(exp!(Underscore)) { None } else { Some(p.parse_expr()?) };
+            ast::InlineAsmOperand::SplitInOut { reg, in_expr: expr, out_expr, late: false }
+        } else {
+            ast::InlineAsmOperand::InOut { reg, expr, late: false }
+        }
+    } else if eat_operand_keyword(p, exp!(Inlateout), asm_macro)? {
+        let reg = parse_reg(p)?;
+        if p.eat_keyword(exp!(Underscore)) {
+            let err = dcx.create_err(errors::AsmUnderscoreInput { span: p.token.span });
+            return Err(err);
+        }
+        let expr = p.parse_expr()?;
+        if p.eat(exp!(FatArrow)) {
+            let out_expr =
+                if p.eat_keyword(exp!(Underscore)) { None } else { Some(p.parse_expr()?) };
+            ast::InlineAsmOperand::SplitInOut { reg, in_expr: expr, out_expr, late: true }
+        } else {
+            ast::InlineAsmOperand::InOut { reg, expr, late: true }
+        }
+    } else if eat_operand_keyword(p, exp!(Label), asm_macro)? {
+        let block = p.parse_block()?;
+        ast::InlineAsmOperand::Label { block }
+    } else if p.eat_keyword(exp!(Const)) {
+        let anon_const = p.parse_expr_anon_const()?;
+        ast::InlineAsmOperand::Const { anon_const }
+    } else if p.eat_keyword(exp!(Sym)) {
+        let expr = p.parse_expr()?;
+        let ast::ExprKind::Path(qself, path) = &expr.kind else {
+            let err = dcx.create_err(errors::AsmSymNoPath { span: expr.span });
+            return Err(err);
+        };
+        let sym =
+            ast::InlineAsmSym { id: ast::DUMMY_NODE_ID, qself: qself.clone(), path: path.clone() };
+        ast::InlineAsmOperand::Sym { sym }
+    } else {
+        return Ok(None);
+    }))
+}
+
+// Public for rustfmt.
+pub fn parse_asm_args<'a>(
+    p: &mut Parser<'a>,
+    sp: Span,
+    asm_macro: AsmMacro,
+) -> PResult<'a, Vec<AsmArg>> {
+    let dcx = p.dcx();
+
+    if p.token == token::Eof {
+        return Err(dcx.create_err(errors::AsmRequiresTemplate { span: sp }));
+    }
+
+    let mut args = Vec::new();
+
+    let attributes = AsmAttrVec::parse(p)?;
+    let first_template = p.parse_expr()?;
+    args.push(AsmArg {
+        span: first_template.span,
+        kind: AsmArgKind::Template(first_template),
+        attributes,
+    });
+
+    let mut allow_templates = true;
+
+    while p.token != token::Eof {
+        if !p.eat(exp!(Comma)) {
+            if allow_templates {
+                // After a template string, we always expect *only* a comma...
+                return Err(dcx.create_err(errors::AsmExpectedComma { span: p.token.span }));
+            } else {
+                // ...after that delegate to `expect` to also include the other expected tokens.
+                return Err(p.expect(exp!(Comma)).err().unwrap());
+            }
+        }
+
+        // Accept trailing commas.
+        if p.token == token::Eof {
+            break;
+        }
+
+        let attributes = AsmAttrVec::parse(p)?;
+        let span_start = p.token.span;
+
+        // Parse `clobber_abi`.
+        if p.eat_keyword(exp!(ClobberAbi)) {
+            allow_templates = false;
+
+            args.push(AsmArg {
+                kind: AsmArgKind::ClobberAbi(parse_clobber_abi(p)?),
+                span: span_start.to(p.prev_token.span),
+                attributes,
+            });
+
+            continue;
+        }
+
+        // Parse `options`.
+        if p.eat_keyword(exp!(Options)) {
+            allow_templates = false;
+
+            args.push(AsmArg {
+                kind: AsmArgKind::Options(parse_options(p, asm_macro)?),
+                span: span_start.to(p.prev_token.span),
+                attributes,
+            });
+
+            continue;
+        }
+
+        // Parse operand names.
+        let name = if p.token.is_ident() && p.look_ahead(1, |t| *t == token::Eq) {
+            let (ident, _) = p.token.ident().unwrap();
+            p.bump();
+            p.expect(exp!(Eq))?;
+            allow_templates = false;
+            Some(ident.name)
+        } else {
+            None
+        };
+
+        if let Some(op) = parse_asm_operand(p, asm_macro)? {
+            allow_templates = false;
+
+            args.push(AsmArg {
+                span: span_start.to(p.prev_token.span),
+                kind: AsmArgKind::Operand(name, op),
+                attributes,
+            });
+        } else if allow_templates {
+            let template = p.parse_expr()?;
+            // If it can't possibly expand to a string, provide diagnostics here to include other
+            // things it could have been.
+            match template.kind {
+                ast::ExprKind::Lit(token_lit)
+                    if matches!(
+                        token_lit.kind,
+                        token::LitKind::Str | token::LitKind::StrRaw(_)
+                    ) => {}
+                ast::ExprKind::MacCall(..) => {}
+                _ => {
+                    let err = dcx.create_err(errors::AsmExpectedOther {
+                        span: template.span,
+                        is_inline_asm: matches!(asm_macro, AsmMacro::Asm),
+                    });
+                    return Err(err);
+                }
+            }
+
+            args.push(AsmArg {
+                span: template.span,
+                kind: AsmArgKind::Template(template),
+                attributes,
+            });
+        } else {
+            p.unexpected_any()?
+        }
+    }
+
+    Ok(args)
+}
+
+fn parse_options<'a>(p: &mut Parser<'a>, asm_macro: AsmMacro) -> PResult<'a, Vec<AsmOption>> {
+    p.expect(exp!(OpenParen))?;
+
+    let mut asm_options = Vec::new();
+
+    while !p.eat(exp!(CloseParen)) {
+        const OPTIONS: [(ExpKeywordPair, ast::InlineAsmOptions); ast::InlineAsmOptions::COUNT] = [
+            (exp!(Pure), ast::InlineAsmOptions::PURE),
+            (exp!(Nomem), ast::InlineAsmOptions::NOMEM),
+            (exp!(Readonly), ast::InlineAsmOptions::READONLY),
+            (exp!(PreservesFlags), ast::InlineAsmOptions::PRESERVES_FLAGS),
+            (exp!(Noreturn), ast::InlineAsmOptions::NORETURN),
+            (exp!(Nostack), ast::InlineAsmOptions::NOSTACK),
+            (exp!(MayUnwind), ast::InlineAsmOptions::MAY_UNWIND),
+            (exp!(AttSyntax), ast::InlineAsmOptions::ATT_SYNTAX),
+            (exp!(Raw), ast::InlineAsmOptions::RAW),
+        ];
+
+        'blk: {
+            for (exp, options) in OPTIONS {
+                // Gives a more accurate list of expected next tokens.
+                let kw_matched = if asm_macro.is_supported_option(options) {
+                    p.eat_keyword(exp)
+                } else {
+                    p.eat_keyword_noexpect(exp.kw)
+                };
+
+                if kw_matched {
+                    let span = p.prev_token.span;
+                    let span_with_comma =
+                        if p.token == token::Comma { span.to(p.token.span) } else { span };
+
+                    asm_options.push(AsmOption { symbol: exp.kw, span, options, span_with_comma });
+                    break 'blk;
+                }
+            }
+
+            return p.unexpected_any();
+        }
+
+        // Allow trailing commas.
+        if p.eat(exp!(CloseParen)) {
+            break;
+        }
+        p.expect(exp!(Comma))?;
+    }
+
+    Ok(asm_options)
+}
+
+fn parse_clobber_abi<'a>(p: &mut Parser<'a>) -> PResult<'a, Vec<(Symbol, Span)>> {
+    p.expect(exp!(OpenParen))?;
+
+    if p.eat(exp!(CloseParen)) {
+        return Err(p.dcx().create_err(errors::NonABI { span: p.token.span }));
+    }
+
+    let mut new_abis = Vec::new();
+    while !p.eat(exp!(CloseParen)) {
+        match p.parse_str_lit() {
+            Ok(str_lit) => {
+                new_abis.push((str_lit.symbol_unescaped, str_lit.span));
+            }
+            Err(opt_lit) => {
+                let span = opt_lit.map_or(p.token.span, |lit| lit.span);
+                return Err(p.dcx().create_err(errors::AsmExpectedStringLiteral { span }));
+            }
+        };
+
+        // Allow trailing commas
+        if p.eat(exp!(CloseParen)) {
+            break;
+        }
+        p.expect(exp!(Comma))?;
+    }
+
+    Ok(new_abis)
+}
+
+fn parse_reg<'a>(p: &mut Parser<'a>) -> PResult<'a, ast::InlineAsmRegOrRegClass> {
+    p.expect(exp!(OpenParen))?;
+    let result = match p.token.uninterpolate().kind {
+        token::Ident(name, IdentIsRaw::No) => ast::InlineAsmRegOrRegClass::RegClass(name),
+        token::Literal(token::Lit { kind: token::LitKind::Str, symbol, suffix: _ }) => {
+            ast::InlineAsmRegOrRegClass::Reg(symbol)
+        }
+        _ => {
+            return Err(p.dcx().create_err(errors::ExpectedRegisterClassOrExplicitRegister {
+                span: p.token.span,
+            }));
+        }
+    };
+    p.bump();
+    p.expect(exp!(CloseParen))?;
+    Ok(result)
+}
diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs
index 6277dde7c97..b49a13ce584 100644
--- a/compiler/rustc_parse/src/parser/diagnostics.rs
+++ b/compiler/rustc_parse/src/parser/diagnostics.rs
@@ -2273,9 +2273,9 @@ impl<'a> Parser<'a> {
                     ),
                     // Also catches `fn foo(&a)`.
                     PatKind::Ref(ref inner_pat, mutab)
-                        if matches!(inner_pat.clone().into_inner().kind, PatKind::Ident(..)) =>
+                        if matches!(inner_pat.clone().kind, PatKind::Ident(..)) =>
                     {
-                        match inner_pat.clone().into_inner().kind {
+                        match inner_pat.clone().kind {
                             PatKind::Ident(_, ident, _) => {
                                 let mutab = mutab.prefix_str();
                                 (
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index 2a7910a6af4..a298c4d4dec 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -834,7 +834,7 @@ impl<'a> Parser<'a> {
         // guides recovery in case we write `&raw expr`.
         if borrow_kind == ast::BorrowKind::Ref
             && mutbl == ast::Mutability::Not
-            && matches!(&expr.kind, ExprKind::Path(None, p) if p.is_ident(kw::Raw))
+            && matches!(&expr.kind, ExprKind::Path(None, p) if *p == kw::Raw)
         {
             self.expected_token_types.insert(TokenType::KwMut);
             self.expected_token_types.insert(TokenType::KwConst);
@@ -1119,7 +1119,7 @@ impl<'a> Parser<'a> {
     /// Parse the field access used in offset_of, matched by `$(e:expr)+`.
     /// Currently returns a list of idents. However, it should be possible in
     /// future to also do array indices, which might be arbitrary expressions.
-    fn parse_floating_field_access(&mut self) -> PResult<'a, P<[Ident]>> {
+    fn parse_floating_field_access(&mut self) -> PResult<'a, Vec<Ident>> {
         let mut fields = Vec::new();
         let mut trailing_dot = None;
 
@@ -3311,26 +3311,44 @@ impl<'a> Parser<'a> {
                             let sm = this.psess.source_map();
                             if let Ok(expr_lines) = sm.span_to_lines(expr_span)
                                 && let Ok(arm_start_lines) = sm.span_to_lines(arm_start_span)
-                                && arm_start_lines.lines[0].end_col == expr_lines.lines[0].end_col
                                 && expr_lines.lines.len() == 2
                             {
-                                // We check whether there's any trailing code in the parse span,
-                                // if there isn't, we very likely have the following:
-                                //
-                                // X |     &Y => "y"
-                                //   |        --    - missing comma
-                                //   |        |
-                                //   |        arrow_span
-                                // X |     &X => "x"
-                                //   |      - ^^ self.token.span
-                                //   |      |
-                                //   |      parsed until here as `"y" & X`
-                                err.span_suggestion_short(
-                                    arm_start_span.shrink_to_hi(),
-                                    "missing a comma here to end this `match` arm",
-                                    ",",
-                                    Applicability::MachineApplicable,
-                                );
+                                if arm_start_lines.lines[0].end_col == expr_lines.lines[0].end_col {
+                                    // We check whether there's any trailing code in the parse span,
+                                    // if there isn't, we very likely have the following:
+                                    //
+                                    // X |     &Y => "y"
+                                    //   |        --    - missing comma
+                                    //   |        |
+                                    //   |        arrow_span
+                                    // X |     &X => "x"
+                                    //   |      - ^^ self.token.span
+                                    //   |      |
+                                    //   |      parsed until here as `"y" & X`
+                                    err.span_suggestion_short(
+                                        arm_start_span.shrink_to_hi(),
+                                        "missing a comma here to end this `match` arm",
+                                        ",",
+                                        Applicability::MachineApplicable,
+                                    );
+                                } else if arm_start_lines.lines[0].end_col + rustc_span::CharPos(1)
+                                    == expr_lines.lines[0].end_col
+                                {
+                                    // similar to the above, but we may typo a `.` or `/` at the end of the line
+                                    let comma_span = arm_start_span
+                                        .shrink_to_hi()
+                                        .with_hi(arm_start_span.hi() + rustc_span::BytePos(1));
+                                    if let Ok(res) = sm.span_to_snippet(comma_span)
+                                        && (res == "." || res == "/")
+                                    {
+                                        err.span_suggestion_short(
+                                            comma_span,
+                                            "you might have meant to write a `,` to end this `match` arm",
+                                            ",",
+                                            Applicability::MachineApplicable,
+                                        );
+                                    }
+                                }
                             }
                         } else {
                             err.span_label(
@@ -3450,10 +3468,8 @@ impl<'a> Parser<'a> {
                 // Detect and recover from `($pat if $cond) => $arm`.
                 // FIXME(guard_patterns): convert this to a normal guard instead
                 let span = pat.span;
-                let ast::PatKind::Paren(subpat) = pat.into_inner().kind else { unreachable!() };
-                let ast::PatKind::Guard(_, mut cond) = subpat.into_inner().kind else {
-                    unreachable!()
-                };
+                let ast::PatKind::Paren(subpat) = pat.kind else { unreachable!() };
+                let ast::PatKind::Guard(_, mut cond) = subpat.kind else { unreachable!() };
                 self.psess.gated_spans.ungate_last(sym::guard_patterns, cond.span);
                 CondChecker::new(self, LetChainsPolicy::AlwaysAllowed).visit_expr(&mut cond);
                 let right = self.prev_token.span;
diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs
index babc55ccc0f..a325c2a57ab 100644
--- a/compiler/rustc_parse/src/parser/item.rs
+++ b/compiler/rustc_parse/src/parser/item.rs
@@ -145,7 +145,7 @@ impl<'a> Parser<'a> {
         {
             let mut item = item.expect("an actual item");
             attrs.prepend_to_nt_inner(&mut item.attrs);
-            return Ok(Some(item.into_inner()));
+            return Ok(Some(*item));
         }
 
         self.collect_tokens(None, attrs, force_collect, |this, mut attrs| {
@@ -637,7 +637,7 @@ impl<'a> Parser<'a> {
                     self.dcx().emit_err(errors::MissingForInTraitImpl { span: missing_for_span });
                 }
 
-                let ty_first = ty_first.into_inner();
+                let ty_first = *ty_first;
                 let path = match ty_first.kind {
                     // This notably includes paths passed through `ty` macro fragments (#46438).
                     TyKind::Path(None, path) => path,
@@ -1577,7 +1577,7 @@ impl<'a> Parser<'a> {
         };
 
         let enum_definition = EnumDef { variants: variants.into_iter().flatten().collect() };
-        Ok(ItemKind::Enum(ident, enum_definition, generics))
+        Ok(ItemKind::Enum(ident, generics, enum_definition))
     }
 
     fn parse_enum_variant(&mut self, span: Span) -> PResult<'a, Option<Variant>> {
@@ -1732,7 +1732,7 @@ impl<'a> Parser<'a> {
             return Err(self.dcx().create_err(err));
         };
 
-        Ok(ItemKind::Struct(ident, vdata, generics))
+        Ok(ItemKind::Struct(ident, generics, vdata))
     }
 
     /// Parses `union Foo { ... }`.
@@ -1764,7 +1764,7 @@ impl<'a> Parser<'a> {
             return Err(err);
         };
 
-        Ok(ItemKind::Union(ident, vdata, generics))
+        Ok(ItemKind::Union(ident, generics, vdata))
     }
 
     /// This function parses the fields of record structs:
diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs
index 968376678f3..b2e90251367 100644
--- a/compiler/rustc_parse/src/parser/mod.rs
+++ b/compiler/rustc_parse/src/parser/mod.rs
@@ -1,3 +1,4 @@
+pub mod asm;
 pub mod attr;
 mod attr_wrapper;
 mod diagnostics;
diff --git a/compiler/rustc_parse/src/parser/pat.rs b/compiler/rustc_parse/src/parser/pat.rs
index d6ff80b2eb4..7a226136e23 100644
--- a/compiler/rustc_parse/src/parser/pat.rs
+++ b/compiler/rustc_parse/src/parser/pat.rs
@@ -1086,7 +1086,7 @@ impl<'a> Parser<'a> {
         if matches!(pat.kind, PatKind::Ident(BindingMode(ByRef::Yes(_), Mutability::Mut), ..)) {
             self.psess.gated_spans.gate(sym::mut_ref, pat.span);
         }
-        Ok(pat.into_inner().kind)
+        Ok(pat.kind)
     }
 
     /// Turn all by-value immutable bindings in a pattern into mutable bindings.
diff --git a/compiler/rustc_parse/src/parser/stmt.rs b/compiler/rustc_parse/src/parser/stmt.rs
index 396ded96bde..c37cb0881c3 100644
--- a/compiler/rustc_parse/src/parser/stmt.rs
+++ b/compiler/rustc_parse/src/parser/stmt.rs
@@ -515,8 +515,8 @@ impl<'a> Parser<'a> {
     fn error_block_no_opening_brace_msg(&mut self, msg: Cow<'static, str>) -> Diag<'a> {
         let prev = self.prev_token.span;
         let sp = self.token.span;
-        let mut e = self.dcx().struct_span_err(sp, msg);
-        self.label_expected_raw_ref(&mut e);
+        let mut err = self.dcx().struct_span_err(sp, msg);
+        self.label_expected_raw_ref(&mut err);
 
         let do_not_suggest_help = self.token.is_keyword(kw::In)
             || self.token == token::Colon
@@ -558,20 +558,19 @@ impl<'a> Parser<'a> {
                     stmt.span
                 };
                 self.suggest_fixes_misparsed_for_loop_head(
-                    &mut e,
+                    &mut err,
                     prev.between(sp),
                     stmt_span,
                     &stmt.kind,
                 );
             }
             Err(e) => {
-                self.recover_stmt_(SemiColonMode::Break, BlockMode::Ignore);
-                e.cancel();
+                e.delay_as_bug();
             }
             _ => {}
         }
-        e.span_label(sp, "expected `{`");
-        e
+        err.span_label(sp, "expected `{`");
+        err
     }
 
     fn suggest_fixes_misparsed_for_loop_head(
@@ -714,7 +713,7 @@ impl<'a> Parser<'a> {
 
     /// Parses the rest of a block expression or function body.
     /// Precondition: already parsed the '{'.
-    pub(crate) fn parse_block_tail(
+    pub fn parse_block_tail(
         &mut self,
         lo: Span,
         s: BlockCheckMode,
diff --git a/compiler/rustc_parse/src/parser/tokenstream/tests.rs b/compiler/rustc_parse/src/parser/tokenstream/tests.rs
index aac75323ff3..19b2c98f5af 100644
--- a/compiler/rustc_parse/src/parser/tokenstream/tests.rs
+++ b/compiler/rustc_parse/src/parser/tokenstream/tests.rs
@@ -14,6 +14,10 @@ fn sp(a: u32, b: u32) -> Span {
     Span::with_root_ctxt(BytePos(a), BytePos(b))
 }
 
+fn cmp_token_stream(a: &TokenStream, b: &TokenStream) -> bool {
+    a.len() == b.len() && a.iter().zip(b.iter()).all(|(x, y)| x.eq_unspanned(y))
+}
+
 #[test]
 fn test_concat() {
     create_default_session_globals_then(|| {
@@ -25,7 +29,7 @@ fn test_concat() {
         eq_res.push_stream(test_snd);
         assert_eq!(test_res.iter().count(), 5);
         assert_eq!(eq_res.iter().count(), 5);
-        assert_eq!(test_res.eq_unspanned(&eq_res), true);
+        assert_eq!(cmp_token_stream(&test_res, &eq_res), true);
     })
 }
 
@@ -104,7 +108,7 @@ fn test_dotdotdot() {
         stream.push_tree(TokenTree::token_joint(token::Dot, sp(0, 1)));
         stream.push_tree(TokenTree::token_joint(token::Dot, sp(1, 2)));
         stream.push_tree(TokenTree::token_alone(token::Dot, sp(2, 3)));
-        assert!(stream.eq_unspanned(&string_to_ts("...")));
+        assert!(cmp_token_stream(&stream, &string_to_ts("...")));
         assert_eq!(stream.iter().count(), 1);
     })
 }
diff --git a/compiler/rustc_parse/src/parser/ty.rs b/compiler/rustc_parse/src/parser/ty.rs
index 17481731b11..9ddfc179e9b 100644
--- a/compiler/rustc_parse/src/parser/ty.rs
+++ b/compiler/rustc_parse/src/parser/ty.rs
@@ -7,6 +7,7 @@ use rustc_ast::{
     Pinnedness, PolyTraitRef, PreciseCapturingArg, TraitBoundModifiers, TraitObjectSyntax, Ty,
     TyKind, UnsafeBinderTy,
 };
+use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_errors::{Applicability, Diag, PResult};
 use rustc_span::{ErrorGuaranteed, Ident, Span, kw, sym};
 use thin_vec::{ThinVec, thin_vec};
@@ -104,14 +105,17 @@ fn can_begin_dyn_bound_in_edition_2015(t: &Token) -> bool {
 impl<'a> Parser<'a> {
     /// Parses a type.
     pub fn parse_ty(&mut self) -> PResult<'a, P<Ty>> {
-        self.parse_ty_common(
-            AllowPlus::Yes,
-            AllowCVariadic::No,
-            RecoverQPath::Yes,
-            RecoverReturnSign::Yes,
-            None,
-            RecoverQuestionMark::Yes,
-        )
+        // Make sure deeply nested types don't overflow the stack.
+        ensure_sufficient_stack(|| {
+            self.parse_ty_common(
+                AllowPlus::Yes,
+                AllowCVariadic::No,
+                RecoverQPath::Yes,
+                RecoverReturnSign::Yes,
+                None,
+                RecoverQuestionMark::Yes,
+            )
+        })
     }
 
     pub(super) fn parse_ty_with_generics_recovery(
@@ -404,7 +408,7 @@ impl<'a> Parser<'a> {
         })?;
 
         if ts.len() == 1 && matches!(trailing, Trailing::No) {
-            let ty = ts.into_iter().next().unwrap().into_inner();
+            let ty = ts.into_iter().next().unwrap();
             let maybe_bounds = allow_plus == AllowPlus::Yes && self.token.is_like_plus();
             match ty.kind {
                 // `(TY_BOUND_NOPAREN) + BOUND + ...`.
@@ -420,7 +424,7 @@ impl<'a> Parser<'a> {
                     self.parse_remaining_bounds(bounds, true)
                 }
                 // `(TYPE)`
-                _ => Ok(TyKind::Paren(P(ty))),
+                _ => Ok(TyKind::Paren(ty)),
             }
         } else {
             Ok(TyKind::Tup(ts))
@@ -1295,7 +1299,7 @@ impl<'a> Parser<'a> {
     ) -> PResult<'a, ()> {
         let fn_path_segment = fn_path.segments.last_mut().unwrap();
         let generic_args = if let Some(p_args) = &fn_path_segment.args {
-            p_args.clone().into_inner()
+            *p_args.clone()
         } else {
             // Normally it wouldn't come here because the upstream should have parsed
             // generic parameters (otherwise it's impossible to call this function).
diff --git a/compiler/rustc_passes/messages.ftl b/compiler/rustc_passes/messages.ftl
index 6d815e510ea..983e562cdf3 100644
--- a/compiler/rustc_passes/messages.ftl
+++ b/compiler/rustc_passes/messages.ftl
@@ -509,23 +509,11 @@ passes_must_not_suspend =
 passes_must_use_no_effect =
     `#[must_use]` has no effect when applied to {$article} {$target}
 
-passes_naked_asm_outside_naked_fn =
-    the `naked_asm!` macro can only be used in functions marked with `#[unsafe(naked)]`
-
-passes_naked_functions_asm_block =
-    naked functions must contain a single `naked_asm!` invocation
-    .label_multiple_asm = multiple `naked_asm!` invocations are not allowed in naked functions
-    .label_non_asm = not allowed in naked functions
-
 passes_naked_functions_incompatible_attribute =
     attribute incompatible with `#[unsafe(naked)]`
     .label = the `{$attr}` attribute is incompatible with `#[unsafe(naked)]`
     .naked_attribute = function marked with `#[unsafe(naked)]` here
 
-passes_naked_functions_must_naked_asm =
-    the `asm!` macro is not allowed in naked functions
-    .label = consider using the `naked_asm!` macro instead
-
 passes_no_link =
     attribute should be applied to an `extern crate` item
     .label = not an `extern crate` item
@@ -556,9 +544,6 @@ passes_no_mangle_foreign =
     .note = symbol names in extern blocks are not mangled
     .suggestion = remove this attribute
 
-passes_no_patterns =
-    patterns not allowed in naked function parameters
-
 passes_no_sanitize =
     `#[no_sanitize({$attr_str})]` should be applied to {$accepted_kind}
     .label = not {$accepted_kind}
@@ -606,10 +591,6 @@ passes_panic_unwind_without_std =
     .note = since the core library is usually precompiled with panic="unwind", rebuilding your crate with panic="abort" may not be enough to fix the problem
     .help = using nightly cargo, use -Zbuild-std with panic="abort" to avoid unwinding
 
-passes_params_not_allowed =
-    referencing function parameters is not allowed in naked functions
-    .help = follow the calling convention in asm block to use parameters
-
 passes_parent_info =
     {$num ->
       [one] {$descr}
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index 5c0d0cf4796..dc29b03083f 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -35,7 +35,7 @@ use rustc_session::lint::builtin::{
     UNKNOWN_OR_MALFORMED_DIAGNOSTIC_ATTRIBUTES, UNUSED_ATTRIBUTES,
 };
 use rustc_session::parse::feature_err;
-use rustc_span::{BytePos, DUMMY_SP, Span, Symbol, edition, kw, sym};
+use rustc_span::{BytePos, DUMMY_SP, Span, Symbol, edition, sym};
 use rustc_trait_selection::error_reporting::InferCtxtErrorExt;
 use rustc_trait_selection::infer::{TyCtxtInferExt, ValuePairs};
 use rustc_trait_selection::traits::ObligationCtxt;
@@ -255,7 +255,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             self.check_generic_attr(hir_id, attr, target, Target::Fn);
                             self.check_proc_macro(hir_id, target, ProcMacroKind::Derive)
                         }
-                        [sym::autodiff, ..] => {
+                        [sym::autodiff_forward, ..] | [sym::autodiff_reverse, ..] => {
                             self.check_autodiff(hir_id, attr, span, target)
                         }
                         [sym::coroutine, ..] => {
@@ -802,7 +802,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
         match target {
             Target::Struct => {
                 if let Some(ItemLike::Item(hir::Item {
-                    kind: hir::ItemKind::Struct(_, hir::VariantData::Struct { fields, .. }, _),
+                    kind: hir::ItemKind::Struct(_, _, hir::VariantData::Struct { fields, .. }),
                     ..
                 })) = item
                     && !fields.is_empty()
@@ -936,7 +936,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
         let span = meta.name_value_literal_span().unwrap_or_else(|| meta.span());
         let attr_str =
             &format!("`#[doc(alias{})]`", if is_list { "(\"...\")" } else { " = \"...\"" });
-        if doc_alias == kw::Empty {
+        if doc_alias == sym::empty {
             tcx.dcx().emit_err(errors::DocAliasEmpty { span, attr_str });
             return;
         }
@@ -1068,7 +1068,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
         }
 
         let doc_keyword = match meta.value_str() {
-            Some(value) if value != kw::Empty => value,
+            Some(value) if value != sym::empty => value,
             _ => return self.doc_attr_str_error(meta, "keyword"),
         };
 
@@ -1130,12 +1130,12 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
             return;
         };
         match item.kind {
-            ItemKind::Enum(_, _, generics) | ItemKind::Struct(_, _, generics)
+            ItemKind::Enum(_, generics, _) | ItemKind::Struct(_, generics, _)
                 if generics.params.len() != 0 => {}
             ItemKind::Trait(_, _, _, generics, _, items)
                 if generics.params.len() != 0
                     || items.iter().any(|item| matches!(item.kind, AssocItemKind::Type)) => {}
-            ItemKind::TyAlias(_, _, generics) if generics.params.len() != 0 => {}
+            ItemKind::TyAlias(_, generics, _) if generics.params.len() != 0 => {}
             _ => {
                 self.dcx().emit_err(errors::DocSearchUnboxInvalid { span: meta.span() });
             }
@@ -1266,13 +1266,17 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
         true
     }
 
-    /// Checks that `doc(test(...))` attribute contains only valid attributes. Returns `true` if
-    /// valid.
-    fn check_test_attr(&self, meta: &MetaItemInner, hir_id: HirId) {
+    /// Checks that `doc(test(...))` attribute contains only valid attributes and are at the right place.
+    fn check_test_attr(&self, attr: &Attribute, meta: &MetaItemInner, hir_id: HirId) {
         if let Some(metas) = meta.meta_item_list() {
             for i_meta in metas {
                 match (i_meta.name(), i_meta.meta_item()) {
-                    (Some(sym::attr | sym::no_crate_inject), _) => {}
+                    (Some(sym::attr), _) => {
+                        // Allowed everywhere like `#[doc]`
+                    }
+                    (Some(sym::no_crate_inject), _) => {
+                        self.check_attr_crate_level(attr, meta, hir_id);
+                    }
                     (_, Some(m)) => {
                         self.tcx.emit_node_span_lint(
                             INVALID_DOC_ATTRIBUTES,
@@ -1359,9 +1363,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                         }
 
                         Some(sym::test) => {
-                            if self.check_attr_crate_level(attr, meta, hir_id) {
-                                self.check_test_attr(meta, hir_id);
-                            }
+                            self.check_test_attr(attr, meta, hir_id);
                         }
 
                         Some(
@@ -2312,7 +2314,9 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
     }
 
     fn check_macro_use(&self, hir_id: HirId, attr: &Attribute, target: Target) {
-        let name = attr.name().unwrap();
+        let Some(name) = attr.name() else {
+            return;
+        };
         match target {
             Target::ExternCrate | Target::Mod => {}
             _ => {
@@ -2792,7 +2796,7 @@ impl<'tcx> Visitor<'tcx> for CheckAttrVisitor<'tcx> {
 }
 
 fn is_c_like_enum(item: &Item<'_>) -> bool {
-    if let ItemKind::Enum(_, ref def, _) = item.kind {
+    if let ItemKind::Enum(_, _, ref def) = item.kind {
         for variant in def.variants {
             match variant.data {
                 hir::VariantData::Unit(..) => { /* continue */ }
diff --git a/compiler/rustc_passes/src/check_export.rs b/compiler/rustc_passes/src/check_export.rs
index 2bb698689be..b9a3849f32f 100644
--- a/compiler/rustc_passes/src/check_export.rs
+++ b/compiler/rustc_passes/src/check_export.rs
@@ -132,7 +132,7 @@ impl<'tcx> Visitor<'tcx> for ExportableItemCollector<'tcx> {
                 self.add_exportable(def_id);
             }
             hir::ItemKind::Use(path, _) => {
-                for res in &path.res {
+                for res in path.res.present_items() {
                     // Only local items are exportable.
                     if let Some(res_id) = res.opt_def_id()
                         && let Some(res_id) = res_id.as_local()
diff --git a/compiler/rustc_passes/src/dead.rs b/compiler/rustc_passes/src/dead.rs
index 0060e726a8e..6b82252f32c 100644
--- a/compiler/rustc_passes/src/dead.rs
+++ b/compiler/rustc_passes/src/dead.rs
@@ -8,12 +8,13 @@ use std::mem;
 use hir::ItemKind;
 use hir::def_id::{LocalDefIdMap, LocalDefIdSet};
 use rustc_abi::FieldIdx;
+use rustc_data_structures::fx::FxIndexSet;
 use rustc_data_structures::unord::UnordSet;
 use rustc_errors::MultiSpan;
 use rustc_hir::def::{CtorOf, DefKind, Res};
 use rustc_hir::def_id::{DefId, LocalDefId, LocalModDefId};
 use rustc_hir::intravisit::{self, Visitor};
-use rustc_hir::{self as hir, Node, PatKind, TyKind};
+use rustc_hir::{self as hir, Node, PatKind, QPath, TyKind};
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::middle::privacy::Level;
 use rustc_middle::query::Providers;
@@ -44,15 +45,20 @@ fn should_explore(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     )
 }
 
-fn ty_ref_to_pub_struct(tcx: TyCtxt<'_>, ty: &hir::Ty<'_>) -> bool {
-    if let TyKind::Path(hir::QPath::Resolved(_, path)) = ty.kind
-        && let Res::Def(def_kind, def_id) = path.res
-        && def_id.is_local()
-        && matches!(def_kind, DefKind::Struct | DefKind::Enum | DefKind::Union)
-    {
-        tcx.visibility(def_id).is_public()
-    } else {
-        true
+/// Returns the local def id of the ADT if the given ty refers to a local one.
+fn local_adt_def_of_ty<'tcx>(ty: &hir::Ty<'tcx>) -> Option<LocalDefId> {
+    match ty.kind {
+        TyKind::Path(QPath::Resolved(_, path)) => {
+            if let Res::Def(def_kind, def_id) = path.res
+                && let Some(local_def_id) = def_id.as_local()
+                && matches!(def_kind, DefKind::Struct | DefKind::Enum | DefKind::Union)
+            {
+                Some(local_def_id)
+            } else {
+                None
+            }
+        }
+        _ => None,
     }
 }
 
@@ -78,7 +84,7 @@ struct MarkSymbolVisitor<'tcx> {
     // maps from ADTs to ignored derived traits (e.g. Debug and Clone)
     // and the span of their respective impl (i.e., part of the derive
     // macro)
-    ignored_derived_traits: LocalDefIdMap<Vec<(DefId, DefId)>>,
+    ignored_derived_traits: LocalDefIdMap<FxIndexSet<(DefId, DefId)>>,
 }
 
 impl<'tcx> MarkSymbolVisitor<'tcx> {
@@ -360,7 +366,7 @@ impl<'tcx> MarkSymbolVisitor<'tcx> {
                 && let Some(fn_sig) =
                     self.tcx.hir_fn_sig_by_hir_id(self.tcx.local_def_id_to_hir_id(local_def_id))
                 && matches!(fn_sig.decl.implicit_self, hir::ImplicitSelfKind::None)
-                && let TyKind::Path(hir::QPath::Resolved(_, path)) =
+                && let TyKind::Path(QPath::Resolved(_, path)) =
                     self.tcx.hir_expect_item(local_impl_of).expect_impl().self_ty.kind
                 && let Res::Def(def_kind, did) = path.res
             {
@@ -388,7 +394,7 @@ impl<'tcx> MarkSymbolVisitor<'tcx> {
                     self.ignored_derived_traits
                         .entry(adt_def_id)
                         .or_default()
-                        .push((trait_of, impl_of));
+                        .insert((trait_of, impl_of));
                 }
                 return true;
             }
@@ -420,51 +426,22 @@ impl<'tcx> MarkSymbolVisitor<'tcx> {
                     intravisit::walk_item(self, item)
                 }
                 hir::ItemKind::ForeignMod { .. } => {}
-                hir::ItemKind::Trait(..) => {
-                    for &impl_def_id in self.tcx.local_trait_impls(item.owner_id.def_id) {
-                        if let ItemKind::Impl(impl_ref) = self.tcx.hir_expect_item(impl_def_id).kind
-                        {
-                            // skip items
-                            // mark dependent traits live
-                            intravisit::walk_generics(self, impl_ref.generics);
-                            // mark dependent parameters live
-                            intravisit::walk_path(self, impl_ref.of_trait.unwrap().path);
+                hir::ItemKind::Trait(.., trait_item_refs) => {
+                    // mark assoc ty live if the trait is live
+                    for trait_item in trait_item_refs {
+                        if matches!(trait_item.kind, hir::AssocItemKind::Type) {
+                            self.check_def_id(trait_item.id.owner_id.to_def_id());
                         }
                     }
-
                     intravisit::walk_item(self, item)
                 }
                 _ => intravisit::walk_item(self, item),
             },
             Node::TraitItem(trait_item) => {
-                // mark corresponding ImplTerm live
+                // mark the trait live
                 let trait_item_id = trait_item.owner_id.to_def_id();
                 if let Some(trait_id) = self.tcx.trait_of_item(trait_item_id) {
-                    // mark the trait live
                     self.check_def_id(trait_id);
-
-                    for impl_id in self.tcx.all_impls(trait_id) {
-                        if let Some(local_impl_id) = impl_id.as_local()
-                            && let ItemKind::Impl(impl_ref) =
-                                self.tcx.hir_expect_item(local_impl_id).kind
-                        {
-                            if !matches!(trait_item.kind, hir::TraitItemKind::Type(..))
-                                && !ty_ref_to_pub_struct(self.tcx, impl_ref.self_ty)
-                            {
-                                // skip methods of private ty,
-                                // they would be solved in `solve_rest_impl_items`
-                                continue;
-                            }
-
-                            // mark self_ty live
-                            intravisit::walk_unambig_ty(self, impl_ref.self_ty);
-                            if let Some(&impl_item_id) =
-                                self.tcx.impl_item_implementor_ids(impl_id).get(&trait_item_id)
-                            {
-                                self.check_def_id(impl_item_id);
-                            }
-                        }
-                    }
                 }
                 intravisit::walk_trait_item(self, trait_item);
             }
@@ -508,48 +485,58 @@ impl<'tcx> MarkSymbolVisitor<'tcx> {
         }
     }
 
-    fn solve_rest_impl_items(&mut self, mut unsolved_impl_items: Vec<(hir::ItemId, LocalDefId)>) {
-        let mut ready;
-        (ready, unsolved_impl_items) =
-            unsolved_impl_items.into_iter().partition(|&(impl_id, impl_item_id)| {
-                self.impl_item_with_used_self(impl_id, impl_item_id)
-            });
-
-        while !ready.is_empty() {
-            self.worklist =
-                ready.into_iter().map(|(_, id)| (id, ComesFromAllowExpect::No)).collect();
-            self.mark_live_symbols();
-
-            (ready, unsolved_impl_items) =
-                unsolved_impl_items.into_iter().partition(|&(impl_id, impl_item_id)| {
-                    self.impl_item_with_used_self(impl_id, impl_item_id)
-                });
+    /// Returns whether `local_def_id` is potentially alive or not.
+    /// `local_def_id` points to an impl or an impl item,
+    /// both impl and impl item that may be passed to this function are of a trait,
+    /// and added into the unsolved_items during `create_and_seed_worklist`
+    fn check_impl_or_impl_item_live(
+        &mut self,
+        impl_id: hir::ItemId,
+        local_def_id: LocalDefId,
+    ) -> bool {
+        if self.should_ignore_item(local_def_id.to_def_id()) {
+            return false;
         }
-    }
 
-    fn impl_item_with_used_self(&mut self, impl_id: hir::ItemId, impl_item_id: LocalDefId) -> bool {
-        if let TyKind::Path(hir::QPath::Resolved(_, path)) =
-            self.tcx.hir_item(impl_id).expect_impl().self_ty.kind
-            && let Res::Def(def_kind, def_id) = path.res
-            && let Some(local_def_id) = def_id.as_local()
-            && matches!(def_kind, DefKind::Struct | DefKind::Enum | DefKind::Union)
-        {
-            if self.tcx.visibility(impl_item_id).is_public() {
-                // for the public method, we don't know the trait item is used or not,
-                // so we mark the method live if the self is used
-                return self.live_symbols.contains(&local_def_id);
+        let trait_def_id = match self.tcx.def_kind(local_def_id) {
+            // assoc impl items of traits are live if the corresponding trait items are live
+            DefKind::AssocFn => self.tcx.associated_item(local_def_id).trait_item_def_id,
+            // impl items are live if the corresponding traits are live
+            DefKind::Impl { of_trait: true } => self
+                .tcx
+                .impl_trait_ref(impl_id.owner_id.def_id)
+                .and_then(|trait_ref| Some(trait_ref.skip_binder().def_id)),
+            _ => None,
+        };
+
+        if let Some(trait_def_id) = trait_def_id {
+            if let Some(trait_def_id) = trait_def_id.as_local()
+                && !self.live_symbols.contains(&trait_def_id)
+            {
+                return false;
             }
 
-            if let Some(trait_item_id) = self.tcx.associated_item(impl_item_id).trait_item_def_id
-                && let Some(local_id) = trait_item_id.as_local()
+            // FIXME: legacy logic to check whether the function may construct `Self`,
+            // this can be removed after supporting marking ADTs appearing in patterns
+            // as live, then we can check private impls of public traits directly
+            if let Some(fn_sig) =
+                self.tcx.hir_fn_sig_by_hir_id(self.tcx.local_def_id_to_hir_id(local_def_id))
+                && matches!(fn_sig.decl.implicit_self, hir::ImplicitSelfKind::None)
+                && self.tcx.visibility(trait_def_id).is_public()
             {
-                // for the private method, we can know the trait item is used or not,
-                // so we mark the method live if the self is used and the trait item is used
-                return self.live_symbols.contains(&local_id)
-                    && self.live_symbols.contains(&local_def_id);
+                return true;
             }
         }
-        false
+
+        // The impl or impl item is used if the corresponding trait or trait item is used and the ty is used.
+        if let Some(local_def_id) =
+            local_adt_def_of_ty(self.tcx.hir_item(impl_id).expect_impl().self_ty)
+            && !self.live_symbols.contains(&local_def_id)
+        {
+            return false;
+        }
+
+        true
     }
 }
 
@@ -584,7 +571,7 @@ impl<'tcx> Visitor<'tcx> for MarkSymbolVisitor<'tcx> {
 
     fn visit_expr(&mut self, expr: &'tcx hir::Expr<'tcx>) {
         match expr.kind {
-            hir::ExprKind::Path(ref qpath @ hir::QPath::TypeRelative(..)) => {
+            hir::ExprKind::Path(ref qpath @ QPath::TypeRelative(..)) => {
                 let res = self.typeck_results().qpath_res(qpath, expr.hir_id);
                 self.handle_res(res);
             }
@@ -707,7 +694,7 @@ fn has_allow_dead_code_or_lang_attr(
             // #[used], #[no_mangle], #[export_name], etc also keeps the item alive
             // forcefully, e.g., for placing it in a specific section.
             cg_attrs.contains_extern_indicator()
-                || cg_attrs.flags.contains(CodegenFnAttrFlags::USED)
+                || cg_attrs.flags.contains(CodegenFnAttrFlags::USED_COMPILER)
                 || cg_attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER)
         }
     }
@@ -738,7 +725,7 @@ fn check_item<'tcx>(
     tcx: TyCtxt<'tcx>,
     worklist: &mut Vec<(LocalDefId, ComesFromAllowExpect)>,
     struct_constructors: &mut LocalDefIdMap<LocalDefId>,
-    unsolved_impl_items: &mut Vec<(hir::ItemId, LocalDefId)>,
+    unsolved_items: &mut Vec<(hir::ItemId, LocalDefId)>,
     id: hir::ItemId,
 ) {
     let allow_dead_code = has_allow_dead_code_or_lang_attr(tcx, id.owner_id.def_id);
@@ -749,7 +736,7 @@ fn check_item<'tcx>(
     match tcx.def_kind(id.owner_id) {
         DefKind::Enum => {
             let item = tcx.hir_item(id);
-            if let hir::ItemKind::Enum(_, ref enum_def, _) = item.kind {
+            if let hir::ItemKind::Enum(_, _, ref enum_def) = item.kind {
                 if let Some(comes_from_allow) = allow_dead_code {
                     worklist.extend(
                         enum_def.variants.iter().map(|variant| (variant.def_id, comes_from_allow)),
@@ -764,47 +751,39 @@ fn check_item<'tcx>(
             }
         }
         DefKind::Impl { of_trait } => {
-            // get DefIds from another query
-            let local_def_ids = tcx
-                .associated_item_def_ids(id.owner_id)
-                .iter()
-                .filter_map(|def_id| def_id.as_local());
-
-            let ty_is_pub = ty_ref_to_pub_struct(tcx, tcx.hir_item(id).expect_impl().self_ty);
-
-            // And we access the Map here to get HirId from LocalDefId
-            for local_def_id in local_def_ids {
-                // check the function may construct Self
-                let mut may_construct_self = false;
-                if let Some(fn_sig) =
-                    tcx.hir_fn_sig_by_hir_id(tcx.local_def_id_to_hir_id(local_def_id))
-                {
-                    may_construct_self =
-                        matches!(fn_sig.decl.implicit_self, hir::ImplicitSelfKind::None);
-                }
+            if let Some(comes_from_allow) =
+                has_allow_dead_code_or_lang_attr(tcx, id.owner_id.def_id)
+            {
+                worklist.push((id.owner_id.def_id, comes_from_allow));
+            } else if of_trait {
+                unsolved_items.push((id, id.owner_id.def_id));
+            }
 
-                // for trait impl blocks,
-                // mark the method live if the self_ty is public,
-                // or the method is public and may construct self
-                if of_trait
-                    && (!matches!(tcx.def_kind(local_def_id), DefKind::AssocFn)
-                        || tcx.visibility(local_def_id).is_public()
-                            && (ty_is_pub || may_construct_self))
-                {
-                    worklist.push((local_def_id, ComesFromAllowExpect::No));
-                } else if let Some(comes_from_allow) =
-                    has_allow_dead_code_or_lang_attr(tcx, local_def_id)
+            for def_id in tcx.associated_item_def_ids(id.owner_id) {
+                let local_def_id = def_id.expect_local();
+
+                if let Some(comes_from_allow) = has_allow_dead_code_or_lang_attr(tcx, local_def_id)
                 {
                     worklist.push((local_def_id, comes_from_allow));
                 } else if of_trait {
-                    // private method || public method not constructs self
-                    unsolved_impl_items.push((id, local_def_id));
+                    // FIXME: This condition can be removed
+                    // if we support dead check for assoc consts and tys.
+                    if !matches!(tcx.def_kind(local_def_id), DefKind::AssocFn) {
+                        worklist.push((local_def_id, ComesFromAllowExpect::No));
+                    } else {
+                        // We only care about associated items of traits,
+                        // because they cannot be visited directly,
+                        // so we later mark them as live if their corresponding traits
+                        // or trait items and self types are both live,
+                        // but inherent associated items can be visited and marked directly.
+                        unsolved_items.push((id, local_def_id));
+                    }
                 }
             }
         }
         DefKind::Struct => {
             let item = tcx.hir_item(id);
-            if let hir::ItemKind::Struct(_, ref variant_data, _) = item.kind
+            if let hir::ItemKind::Struct(_, _, ref variant_data) = item.kind
                 && let Some(ctor_def_id) = variant_data.ctor_def_id()
             {
                 struct_constructors.insert(ctor_def_id, item.owner_id.def_id);
@@ -892,8 +871,8 @@ fn create_and_seed_worklist(
 fn live_symbols_and_ignored_derived_traits(
     tcx: TyCtxt<'_>,
     (): (),
-) -> (LocalDefIdSet, LocalDefIdMap<Vec<(DefId, DefId)>>) {
-    let (worklist, struct_constructors, unsolved_impl_items) = create_and_seed_worklist(tcx);
+) -> (LocalDefIdSet, LocalDefIdMap<FxIndexSet<(DefId, DefId)>>) {
+    let (worklist, struct_constructors, mut unsolved_items) = create_and_seed_worklist(tcx);
     let mut symbol_visitor = MarkSymbolVisitor {
         worklist,
         tcx,
@@ -907,7 +886,22 @@ fn live_symbols_and_ignored_derived_traits(
         ignored_derived_traits: Default::default(),
     };
     symbol_visitor.mark_live_symbols();
-    symbol_visitor.solve_rest_impl_items(unsolved_impl_items);
+    let mut items_to_check;
+    (items_to_check, unsolved_items) =
+        unsolved_items.into_iter().partition(|&(impl_id, local_def_id)| {
+            symbol_visitor.check_impl_or_impl_item_live(impl_id, local_def_id)
+        });
+
+    while !items_to_check.is_empty() {
+        symbol_visitor.worklist =
+            items_to_check.into_iter().map(|(_, id)| (id, ComesFromAllowExpect::No)).collect();
+        symbol_visitor.mark_live_symbols();
+
+        (items_to_check, unsolved_items) =
+            unsolved_items.into_iter().partition(|&(impl_id, local_def_id)| {
+                symbol_visitor.check_impl_or_impl_item_live(impl_id, local_def_id)
+            });
+    }
 
     (symbol_visitor.live_symbols, symbol_visitor.ignored_derived_traits)
 }
@@ -921,7 +915,7 @@ struct DeadItem {
 struct DeadVisitor<'tcx> {
     tcx: TyCtxt<'tcx>,
     live_symbols: &'tcx LocalDefIdSet,
-    ignored_derived_traits: &'tcx LocalDefIdMap<Vec<(DefId, DefId)>>,
+    ignored_derived_traits: &'tcx LocalDefIdMap<FxIndexSet<(DefId, DefId)>>,
 }
 
 enum ShouldWarnAboutField {
@@ -1061,7 +1055,7 @@ impl<'tcx> DeadVisitor<'tcx> {
                 let tuple_fields = if let Some(parent_id) = parent_item
                     && let node = tcx.hir_node_by_def_id(parent_id)
                     && let hir::Node::Item(hir::Item {
-                        kind: hir::ItemKind::Struct(_, hir::VariantData::Tuple(fields, _, _), _),
+                        kind: hir::ItemKind::Struct(_, _, hir::VariantData::Tuple(fields, _, _)),
                         ..
                     }) = node
                 {
@@ -1188,19 +1182,15 @@ fn check_mod_deathness(tcx: TyCtxt<'_>, module: LocalModDefId) {
         let def_kind = tcx.def_kind(item.owner_id);
 
         let mut dead_codes = Vec::new();
-        // if we have diagnosed the trait, do not diagnose unused methods
-        if matches!(def_kind, DefKind::Impl { .. })
+        // Only diagnose unused assoc items in inherient impl and used trait,
+        // for unused assoc items in impls of trait,
+        // we have diagnosed them in the trait if they are unused,
+        // for unused assoc items in unused trait,
+        // we have diagnosed the unused trait.
+        if matches!(def_kind, DefKind::Impl { of_trait: false })
             || (def_kind == DefKind::Trait && live_symbols.contains(&item.owner_id.def_id))
         {
             for &def_id in tcx.associated_item_def_ids(item.owner_id.def_id) {
-                // We have diagnosed unused methods in traits
-                if matches!(def_kind, DefKind::Impl { of_trait: true })
-                    && tcx.def_kind(def_id) == DefKind::AssocFn
-                    || def_kind == DefKind::Trait && tcx.def_kind(def_id) != DefKind::AssocFn
-                {
-                    continue;
-                }
-
                 if let Some(local_def_id) = def_id.as_local()
                     && !visitor.is_live_code(local_def_id)
                 {
diff --git a/compiler/rustc_passes/src/errors.rs b/compiler/rustc_passes/src/errors.rs
index 00682a9c7a7..b995781719b 100644
--- a/compiler/rustc_passes/src/errors.rs
+++ b/compiler/rustc_passes/src/errors.rs
@@ -1197,51 +1197,6 @@ pub(crate) struct UnlabeledCfInWhileCondition<'a> {
 }
 
 #[derive(Diagnostic)]
-#[diag(passes_no_patterns)]
-pub(crate) struct NoPatterns {
-    #[primary_span]
-    pub span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(passes_params_not_allowed)]
-#[help]
-pub(crate) struct ParamsNotAllowed {
-    #[primary_span]
-    pub span: Span,
-}
-
-pub(crate) struct NakedFunctionsAsmBlock {
-    pub span: Span,
-    pub multiple_asms: Vec<Span>,
-    pub non_asms: Vec<Span>,
-}
-
-impl<G: EmissionGuarantee> Diagnostic<'_, G> for NakedFunctionsAsmBlock {
-    #[track_caller]
-    fn into_diag(self, dcx: DiagCtxtHandle<'_>, level: Level) -> Diag<'_, G> {
-        let mut diag = Diag::new(dcx, level, fluent::passes_naked_functions_asm_block);
-        diag.span(self.span);
-        diag.code(E0787);
-        for span in self.multiple_asms.iter() {
-            diag.span_label(*span, fluent::passes_label_multiple_asm);
-        }
-        for span in self.non_asms.iter() {
-            diag.span_label(*span, fluent::passes_label_non_asm);
-        }
-        diag
-    }
-}
-
-#[derive(Diagnostic)]
-#[diag(passes_naked_functions_must_naked_asm, code = E0787)]
-pub(crate) struct NakedFunctionsMustNakedAsm {
-    #[primary_span]
-    #[label]
-    pub span: Span,
-}
-
-#[derive(Diagnostic)]
 #[diag(passes_naked_functions_incompatible_attribute, code = E0736)]
 pub(crate) struct NakedFunctionIncompatibleAttribute {
     #[primary_span]
@@ -1253,13 +1208,6 @@ pub(crate) struct NakedFunctionIncompatibleAttribute {
 }
 
 #[derive(Diagnostic)]
-#[diag(passes_naked_asm_outside_naked_fn)]
-pub(crate) struct NakedAsmOutsideNakedFn {
-    #[primary_span]
-    pub span: Span,
-}
-
-#[derive(Diagnostic)]
 #[diag(passes_attr_only_in_functions)]
 pub(crate) struct AttrOnlyInFunctions {
     #[primary_span]
diff --git a/compiler/rustc_passes/src/input_stats.rs b/compiler/rustc_passes/src/input_stats.rs
index 71815448172..6852153a2e7 100644
--- a/compiler/rustc_passes/src/input_stats.rs
+++ b/compiler/rustc_passes/src/input_stats.rs
@@ -426,10 +426,16 @@ impl<'v> hir_visit::Visitor<'v> for StatCollector<'v> {
         hir_visit::walk_fn(self, fk, fd, b, id)
     }
 
-    fn visit_use(&mut self, p: &'v hir::UsePath<'v>, hir_id: HirId) {
+    fn visit_use(&mut self, p: &'v hir::UsePath<'v>, _hir_id: HirId) {
         // This is `visit_use`, but the type is `Path` so record it that way.
         self.record("Path", None, p);
-        hir_visit::walk_use(self, p, hir_id)
+        // Don't call `hir_visit::walk_use(self, p, hir_id)`: it calls
+        // `visit_path` up to three times, once for each namespace result in
+        // `p.res`, by building temporary `Path`s that are not part of the real
+        // HIR, which causes `p` to be double- or triple-counted. Instead just
+        // walk the path internals (i.e. the segments) directly.
+        let hir::Path { span: _, res: _, segments } = *p;
+        ast_visit::walk_list!(self, visit_path_segment, segments);
     }
 
     fn visit_trait_item(&mut self, ti: &'v hir::TraitItem<'v>) {
diff --git a/compiler/rustc_passes/src/lang_items.rs b/compiler/rustc_passes/src/lang_items.rs
index 275714c2d0e..3afed9784de 100644
--- a/compiler/rustc_passes/src/lang_items.rs
+++ b/compiler/rustc_passes/src/lang_items.rs
@@ -307,18 +307,14 @@ impl<'ast, 'tcx> visit::Visitor<'ast> for LanguageItemCollector<'ast, 'tcx> {
         self.parent_item = parent_item;
     }
 
-    fn visit_enum_def(&mut self, enum_definition: &'ast ast::EnumDef) {
-        for variant in &enum_definition.variants {
-            self.check_for_lang(
-                Target::Variant,
-                self.resolver.node_id_to_def_id[&variant.id],
-                &variant.attrs,
-                variant.span,
-                None,
-            );
-        }
-
-        visit::walk_enum_def(self, enum_definition);
+    fn visit_variant(&mut self, variant: &'ast ast::Variant) {
+        self.check_for_lang(
+            Target::Variant,
+            self.resolver.node_id_to_def_id[&variant.id],
+            &variant.attrs,
+            variant.span,
+            None,
+        );
     }
 
     fn visit_assoc_item(&mut self, i: &'ast ast::AssocItem, ctxt: visit::AssocCtxt) {
diff --git a/compiler/rustc_passes/src/lib.rs b/compiler/rustc_passes/src/lib.rs
index f9445485f60..1831f45a9ec 100644
--- a/compiler/rustc_passes/src/lib.rs
+++ b/compiler/rustc_passes/src/lib.rs
@@ -8,13 +8,11 @@
 #![allow(internal_features)]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
-#![feature(box_patterns)]
 #![feature(map_try_insert)]
 #![feature(rustdoc_internals)]
-#![feature(try_blocks)]
 // tidy-alphabetical-end
 
-use rustc_middle::query::Providers;
+use rustc_middle::util::Providers;
 
 pub mod abi_test;
 mod check_attr;
@@ -32,7 +30,6 @@ pub mod layout_test;
 mod lib_features;
 mod liveness;
 pub mod loops;
-mod naked_functions;
 mod reachable;
 pub mod stability;
 mod upvars;
@@ -49,7 +46,6 @@ pub fn provide(providers: &mut Providers) {
     lang_items::provide(providers);
     lib_features::provide(providers);
     loops::provide(providers);
-    naked_functions::provide(providers);
     liveness::provide(providers);
     reachable::provide(providers);
     stability::provide(providers);
diff --git a/compiler/rustc_passes/src/liveness.rs b/compiler/rustc_passes/src/liveness.rs
index 4e9b7fd44d4..763d9fda804 100644
--- a/compiler/rustc_passes/src/liveness.rs
+++ b/compiler/rustc_passes/src/liveness.rs
@@ -122,7 +122,6 @@ enum LiveNodeKind {
     VarDefNode(Span, HirId),
     ClosureNode,
     ExitNode,
-    ErrNode,
 }
 
 fn live_node_kind_to_string(lnk: LiveNodeKind, tcx: TyCtxt<'_>) -> String {
@@ -133,7 +132,6 @@ fn live_node_kind_to_string(lnk: LiveNodeKind, tcx: TyCtxt<'_>) -> String {
         VarDefNode(s, _) => format!("Var def node [{}]", sm.span_to_diagnostic_string(s)),
         ClosureNode => "Closure node".to_owned(),
         ExitNode => "Exit node".to_owned(),
-        ErrNode => "Error node".to_owned(),
     }
 }
 
@@ -492,6 +490,9 @@ struct Liveness<'a, 'tcx> {
 impl<'a, 'tcx> Liveness<'a, 'tcx> {
     fn new(ir: &'a mut IrMaps<'tcx>, body_owner: LocalDefId) -> Liveness<'a, 'tcx> {
         let typeck_results = ir.tcx.typeck(body_owner);
+        // Liveness linting runs after building the THIR. We make several assumptions based on
+        // typeck succeeding, e.g. that breaks and continues are well-formed.
+        assert!(typeck_results.tainted_by_errors.is_none());
         // FIXME(#132279): we're in a body here.
         let typing_env = ty::TypingEnv::non_body_analysis(ir.tcx, body_owner);
         let closure_min_captures = typeck_results.closure_min_captures.get(&body_owner);
@@ -976,8 +977,9 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
                 // Now that we know the label we're going to,
                 // look it up in the continue loop nodes table
                 self.cont_ln.get(&sc).cloned().unwrap_or_else(|| {
-                    self.ir.tcx.dcx().span_delayed_bug(expr.span, "continue to unknown label");
-                    self.ir.add_live_node(ErrNode)
+                    // Liveness linting happens after building the THIR. Bad labels should already
+                    // have been caught.
+                    span_bug!(expr.span, "continue to unknown label");
                 })
             }
 
diff --git a/compiler/rustc_passes/src/reachable.rs b/compiler/rustc_passes/src/reachable.rs
index f0e8fa986fe..7e15267a953 100644
--- a/compiler/rustc_passes/src/reachable.rs
+++ b/compiler/rustc_passes/src/reachable.rs
@@ -427,7 +427,7 @@ fn has_custom_linkage(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
         // FIXME(nbdd0121): `#[used]` are marked as reachable here so it's picked up by
         // `linked_symbols` in cg_ssa. They won't be exported in binary or cdylib due to their
         // `SymbolExportLevel::Rust` export level but may end up being exported in dylibs.
-        || codegen_attrs.flags.contains(CodegenFnAttrFlags::USED)
+        || codegen_attrs.flags.contains(CodegenFnAttrFlags::USED_COMPILER)
         || codegen_attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER)
 }
 
diff --git a/compiler/rustc_passes/src/stability.rs b/compiler/rustc_passes/src/stability.rs
index 9884386d68f..45e26c8999a 100644
--- a/compiler/rustc_passes/src/stability.rs
+++ b/compiler/rustc_passes/src/stability.rs
@@ -430,7 +430,7 @@ impl<'a, 'tcx> Visitor<'tcx> for Annotator<'a, 'tcx> {
                 kind = AnnotationKind::DeprecationProhibited;
                 const_stab_inherit = InheritConstStability::Yes;
             }
-            hir::ItemKind::Struct(_, ref sd, _) => {
+            hir::ItemKind::Struct(_, _, ref sd) => {
                 if let Some(ctor_def_id) = sd.ctor_def_id() {
                     self.annotate(
                         ctor_def_id,
diff --git a/compiler/rustc_pattern_analysis/src/rustc.rs b/compiler/rustc_pattern_analysis/src/rustc.rs
index a89d01dcbbe..46ced7500ea 100644
--- a/compiler/rustc_pattern_analysis/src/rustc.rs
+++ b/compiler/rustc_pattern_analysis/src/rustc.rs
@@ -1,6 +1,5 @@
 use std::fmt;
 use std::iter::once;
-use std::ops::ControlFlow;
 
 use rustc_abi::{FIRST_VARIANT, FieldIdx, Integer, VariantIdx};
 use rustc_arena::DroplessArena;
@@ -12,8 +11,7 @@ use rustc_middle::mir::{self, Const};
 use rustc_middle::thir::{self, Pat, PatKind, PatRange, PatRangeBoundary};
 use rustc_middle::ty::layout::IntegerExt;
 use rustc_middle::ty::{
-    self, FieldDef, OpaqueTypeKey, ScalarInt, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable,
-    TypeVisitableExt, TypeVisitor, VariantDef,
+    self, FieldDef, OpaqueTypeKey, ScalarInt, Ty, TyCtxt, TypeVisitableExt, VariantDef,
 };
 use rustc_middle::{bug, span_bug};
 use rustc_session::lint;
@@ -137,22 +135,11 @@ impl<'p, 'tcx: 'p> RustcPatCtxt<'p, 'tcx> {
     /// Returns the hidden type corresponding to this key if the body under analysis is allowed to
     /// know it.
     fn reveal_opaque_key(&self, key: OpaqueTypeKey<'tcx>) -> Option<Ty<'tcx>> {
-        if let Some(hidden_ty) = self.typeck_results.concrete_opaque_types.get(&key.def_id) {
-            let ty = ty::EarlyBinder::bind(hidden_ty.ty).instantiate(self.tcx, key.args);
-            if ty.visit_with(&mut RecursiveOpaque { def_id: key.def_id.into() }).is_continue() {
-                Some(ty)
-            } else {
-                // HACK: We skip revealing opaque types which recursively expand
-                // to themselves. This is because we may infer hidden types like
-                // `Opaque<T> = Opaque<Opaque<T>>` or `Opaque<T> = Opaque<(T,)>`
-                // in hir typeck.
-                None
-            }
-        } else {
-            None
-        }
+        self.typeck_results
+            .concrete_opaque_types
+            .get(&key.def_id)
+            .map(|x| ty::EarlyBinder::bind(x.ty).instantiate(self.tcx, key.args))
     }
-
     // This can take a non-revealed `Ty` because it reveals opaques itself.
     pub fn is_uninhabited(&self, ty: Ty<'tcx>) -> bool {
         !ty.inhabited_predicate(self.tcx).apply_revealing_opaque(
@@ -262,12 +249,12 @@ impl<'p, 'tcx: 'p> RustcPatCtxt<'p, 'tcx> {
                 ty::Ref(_, rty, _) => reveal_and_alloc(cx, once(*rty)),
                 _ => bug!("Unexpected type for `Ref` constructor: {ty:?}"),
             },
-            Slice(slice) => match *ty.kind() {
-                ty::Slice(ty) | ty::Array(ty, _) => {
+            Slice(slice) => match ty.builtin_index() {
+                Some(ty) => {
                     let arity = slice.arity();
                     reveal_and_alloc(cx, (0..arity).map(|_| ty))
                 }
-                _ => bug!("bad slice pattern {:?} {:?}", ctor, ty),
+                None => bug!("bad slice pattern {:?} {:?}", ctor, ty),
             },
             DerefPattern(pointee_ty) => reveal_and_alloc(cx, once(pointee_ty.inner())),
             Bool(..) | IntRange(..) | F16Range(..) | F32Range(..) | F64Range(..)
@@ -1177,20 +1164,3 @@ fn detect_mixed_deref_pat_ctors<'p, 'tcx>(
     }
     Ok(())
 }
-
-struct RecursiveOpaque {
-    def_id: DefId,
-}
-impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for RecursiveOpaque {
-    type Result = ControlFlow<()>;
-
-    fn visit_ty(&mut self, t: Ty<'tcx>) -> Self::Result {
-        if let ty::Alias(ty::Opaque, alias_ty) = t.kind() {
-            if alias_ty.def_id == self.def_id {
-                return ControlFlow::Break(());
-            }
-        }
-
-        if t.has_opaque_types() { t.super_visit_with(self) } else { ControlFlow::Continue(()) }
-    }
-}
diff --git a/compiler/rustc_privacy/src/lib.rs b/compiler/rustc_privacy/src/lib.rs
index e2dfaec61b3..963f4c77d80 100644
--- a/compiler/rustc_privacy/src/lib.rs
+++ b/compiler/rustc_privacy/src/lib.rs
@@ -599,8 +599,8 @@ impl<'tcx> EmbargoVisitor<'tcx> {
             DefKind::Struct | DefKind::Union => {
                 // While structs and unions have type privacy, their fields do not.
                 let item = self.tcx.hir_expect_item(def_id);
-                if let hir::ItemKind::Struct(_, ref struct_def, _)
-                | hir::ItemKind::Union(_, ref struct_def, _) = item.kind
+                if let hir::ItemKind::Struct(_, _, ref struct_def)
+                | hir::ItemKind::Union(_, _, ref struct_def) = item.kind
                 {
                     for field in struct_def.fields() {
                         let field_vis = self.tcx.local_visibility(field.def_id);
@@ -725,7 +725,7 @@ impl<'tcx> Visitor<'tcx> for EmbargoVisitor<'tcx> {
                     }
                 }
             }
-            hir::ItemKind::Enum(_, ref def, _) => {
+            hir::ItemKind::Enum(_, _, ref def) => {
                 if let Some(item_ev) = item_ev {
                     self.reach(item.owner_id.def_id, item_ev).generics().predicates();
                 }
@@ -763,8 +763,8 @@ impl<'tcx> Visitor<'tcx> for EmbargoVisitor<'tcx> {
                     }
                 }
             }
-            hir::ItemKind::Struct(_, ref struct_def, _)
-            | hir::ItemKind::Union(_, ref struct_def, _) => {
+            hir::ItemKind::Struct(_, _, ref struct_def)
+            | hir::ItemKind::Union(_, _, ref struct_def) => {
                 if let Some(item_ev) = item_ev {
                     self.reach(item.owner_id.def_id, item_ev).generics().predicates();
                     for field in struct_def.fields() {
@@ -868,7 +868,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TestReachabilityVisitor<'a, 'tcx> {
         self.effective_visibility_diagnostic(item.owner_id.def_id);
 
         match item.kind {
-            hir::ItemKind::Enum(_, ref def, _) => {
+            hir::ItemKind::Enum(_, _, ref def) => {
                 for variant in def.variants.iter() {
                     self.effective_visibility_diagnostic(variant.def_id);
                     if let Some(ctor_def_id) = variant.data.ctor_def_id() {
@@ -879,7 +879,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TestReachabilityVisitor<'a, 'tcx> {
                     }
                 }
             }
-            hir::ItemKind::Struct(_, ref def, _) | hir::ItemKind::Union(_, ref def, _) => {
+            hir::ItemKind::Struct(_, _, ref def) | hir::ItemKind::Union(_, _, ref def) => {
                 if let Some(ctor_def_id) = def.ctor_def_id() {
                     self.effective_visibility_diagnostic(ctor_def_id);
                 }
@@ -1651,7 +1651,7 @@ impl<'tcx> PrivateItemsInPublicInterfacesChecker<'_, 'tcx> {
             }
             DefKind::Enum => {
                 let item = tcx.hir_item(id);
-                if let hir::ItemKind::Enum(_, ref def, _) = item.kind {
+                if let hir::ItemKind::Enum(_, _, ref def) = item.kind {
                     self.check_unnameable(item.owner_id.def_id, effective_vis);
 
                     self.check(item.owner_id.def_id, item_visibility, effective_vis)
@@ -1689,8 +1689,8 @@ impl<'tcx> PrivateItemsInPublicInterfacesChecker<'_, 'tcx> {
             // Subitems of structs and unions have their own publicity.
             DefKind::Struct | DefKind::Union => {
                 let item = tcx.hir_item(id);
-                if let hir::ItemKind::Struct(_, ref struct_def, _)
-                | hir::ItemKind::Union(_, ref struct_def, _) = item.kind
+                if let hir::ItemKind::Struct(_, _, ref struct_def)
+                | hir::ItemKind::Union(_, _, ref struct_def) = item.kind
                 {
                     self.check_unnameable(item.owner_id.def_id, effective_vis);
                     self.check(item.owner_id.def_id, item_visibility, effective_vis)
diff --git a/compiler/rustc_proc_macro/Cargo.toml b/compiler/rustc_proc_macro/Cargo.toml
new file mode 100644
index 00000000000..4a7c0d78ede
--- /dev/null
+++ b/compiler/rustc_proc_macro/Cargo.toml
@@ -0,0 +1,21 @@
+# We need to use a separate crate including library/proc_macro as opposed to a
+# direct path dependency on library/proc_macro because doing the latter will
+# cause two copies of libproc_macro.rlib to end up in the sysroot, breaking
+# proc-macro crates. In addition it confuses the workspace_members function of
+# bootstrap.
+
+[package]
+name = "rustc_proc_macro"
+version = "0.0.0"
+edition = "2024"
+
+[lib]
+path = "../../library/proc_macro/src/lib.rs"
+test = false
+doctest = false
+
+[dependencies]
+rustc-literal-escaper = "0.0.2"
+
+[features]
+rustc-dep-of-std = []
diff --git a/compiler/rustc_query_system/src/dep_graph/dep_node.rs b/compiler/rustc_query_system/src/dep_graph/dep_node.rs
index c0b3bd43e25..bdd1d5f3e88 100644
--- a/compiler/rustc_query_system/src/dep_graph/dep_node.rs
+++ b/compiler/rustc_query_system/src/dep_graph/dep_node.rs
@@ -178,9 +178,7 @@ pub trait DepNodeParams<Tcx: DepContext>: fmt::Debug + Sized {
         panic!("Not implemented. Accidentally called on anonymous node?")
     }
 
-    fn to_debug_str(&self, _: Tcx) -> String {
-        format!("{self:?}")
-    }
+    fn to_debug_str(&self, tcx: Tcx) -> String;
 
     /// This method tries to recover the query key from the given `DepNode`,
     /// something which is needed when forcing `DepNode`s during red-green
@@ -210,8 +208,11 @@ where
     }
 
     #[inline(always)]
-    default fn to_debug_str(&self, _: Tcx) -> String {
-        format!("{:?}", *self)
+    default fn to_debug_str(&self, tcx: Tcx) -> String {
+        // Make sure to print dep node params with reduced queries since printing
+        // may themselves call queries, which may lead to (possibly untracked!)
+        // query cycles.
+        tcx.with_reduced_queries(|| format!("{self:?}"))
     }
 
     #[inline(always)]
diff --git a/compiler/rustc_query_system/src/dep_graph/mod.rs b/compiler/rustc_query_system/src/dep_graph/mod.rs
index 89d1db87809..512034a8b2f 100644
--- a/compiler/rustc_query_system/src/dep_graph/mod.rs
+++ b/compiler/rustc_query_system/src/dep_graph/mod.rs
@@ -88,6 +88,8 @@ pub trait DepContext: Copy {
             f(self, dep_node)
         }
     }
+
+    fn with_reduced_queries<T>(self, _: impl FnOnce() -> T) -> T;
 }
 
 pub trait Deps: DynSync {
diff --git a/compiler/rustc_query_system/src/lib.rs b/compiler/rustc_query_system/src/lib.rs
index d36cb6f0e5b..7fa643d91aa 100644
--- a/compiler/rustc_query_system/src/lib.rs
+++ b/compiler/rustc_query_system/src/lib.rs
@@ -2,7 +2,6 @@
 #![allow(internal_features)]
 #![feature(assert_matches)]
 #![feature(core_intrinsics)]
-#![feature(dropck_eyepatch)]
 #![feature(min_specialization)]
 // tidy-alphabetical-end
 
diff --git a/compiler/rustc_resolve/src/build_reduced_graph.rs b/compiler/rustc_resolve/src/build_reduced_graph.rs
index 3460c53782f..c30ed781f35 100644
--- a/compiler/rustc_resolve/src/build_reduced_graph.rs
+++ b/compiler/rustc_resolve/src/build_reduced_graph.rs
@@ -823,7 +823,7 @@ impl<'a, 'ra, 'tcx> BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
             }
 
             // These items live in both the type and value namespaces.
-            ItemKind::Struct(ident, ref vdata, _) => {
+            ItemKind::Struct(ident, _, ref vdata) => {
                 self.build_reduced_graph_for_struct_variant(
                     vdata.fields(),
                     ident,
@@ -874,7 +874,7 @@ impl<'a, 'ra, 'tcx> BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
                 }
             }
 
-            ItemKind::Union(ident, ref vdata, _) => {
+            ItemKind::Union(ident, _, ref vdata) => {
                 self.build_reduced_graph_for_struct_variant(
                     vdata.fields(),
                     ident,
diff --git a/compiler/rustc_resolve/src/check_unused.rs b/compiler/rustc_resolve/src/check_unused.rs
index e97233e97ce..2e870c47f8e 100644
--- a/compiler/rustc_resolve/src/check_unused.rs
+++ b/compiler/rustc_resolve/src/check_unused.rs
@@ -100,6 +100,21 @@ impl<'a, 'ra, 'tcx> UnusedImportCheckVisitor<'a, 'ra, 'tcx> {
         }
     }
 
+    fn check_use_tree(&mut self, use_tree: &'a ast::UseTree, id: ast::NodeId) {
+        if self.r.effective_visibilities.is_exported(self.r.local_def_id(id)) {
+            self.check_import_as_underscore(use_tree, id);
+            return;
+        }
+
+        if let ast::UseTreeKind::Nested { ref items, .. } = use_tree.kind {
+            if items.is_empty() {
+                self.unused_import(self.base_id).add(id);
+            }
+        } else {
+            self.check_import(id);
+        }
+    }
+
     fn unused_import(&mut self, id: ast::NodeId) -> &mut UnusedImport {
         let use_tree_id = self.base_id;
         let use_tree = self.base_use_tree.unwrap().clone();
@@ -118,9 +133,10 @@ impl<'a, 'ra, 'tcx> UnusedImportCheckVisitor<'a, 'ra, 'tcx> {
             ast::UseTreeKind::Simple(Some(ident)) => {
                 if ident.name == kw::Underscore
                     && !self.r.import_res_map.get(&id).is_some_and(|per_ns| {
-                        per_ns.iter().filter_map(|res| res.as_ref()).any(|res| {
-                            matches!(res, Res::Def(DefKind::Trait | DefKind::TraitAlias, _))
-                        })
+                        matches!(
+                            per_ns.type_ns,
+                            Some(Res::Def(DefKind::Trait | DefKind::TraitAlias, _))
+                        )
                     })
                 {
                     self.unused_import(self.base_id).add(id);
@@ -154,6 +170,7 @@ impl<'a, 'ra, 'tcx> UnusedImportCheckVisitor<'a, 'ra, 'tcx> {
                         extern_crate.id,
                         span,
                         BuiltinLintDiag::UnusedExternCrate {
+                            span: extern_crate.span,
                             removal_span: extern_crate.span_with_attributes,
                         },
                     );
@@ -192,6 +209,16 @@ impl<'a, 'ra, 'tcx> UnusedImportCheckVisitor<'a, 'ra, 'tcx> {
                 continue;
             }
 
+            let module = self
+                .r
+                .get_nearest_non_block_module(self.r.local_def_id(extern_crate.id).to_def_id());
+            if module.no_implicit_prelude {
+                // If the module has `no_implicit_prelude`, then we don't suggest
+                // replacing the extern crate with a use, as it would not be
+                // inserted into the prelude. User writes `extern` style deliberately.
+                continue;
+            }
+
             let vis_span = extern_crate
                 .vis_span
                 .find_ancestor_inside(extern_crate.span)
@@ -213,13 +240,21 @@ impl<'a, 'ra, 'tcx> UnusedImportCheckVisitor<'a, 'ra, 'tcx> {
 
 impl<'a, 'ra, 'tcx> Visitor<'a> for UnusedImportCheckVisitor<'a, 'ra, 'tcx> {
     fn visit_item(&mut self, item: &'a ast::Item) {
-        match item.kind {
+        self.item_span = item.span_with_attributes();
+        match &item.kind {
             // Ignore is_public import statements because there's no way to be sure
             // whether they're used or not. Also ignore imports with a dummy span
             // because this means that they were generated in some fashion by the
             // compiler and we don't need to consider them.
             ast::ItemKind::Use(..) if item.span.is_dummy() => return,
-            ast::ItemKind::ExternCrate(orig_name, ident) => {
+            // Use the base UseTree's NodeId as the item id
+            // This allows the grouping of all the lints in the same item
+            ast::ItemKind::Use(use_tree) => {
+                self.base_id = item.id;
+                self.base_use_tree = Some(use_tree);
+                self.check_use_tree(use_tree, item.id);
+            }
+            &ast::ItemKind::ExternCrate(orig_name, ident) => {
                 self.extern_crate_items.push(ExternCrateToLint {
                     id: item.id,
                     span: item.span,
@@ -233,32 +268,12 @@ impl<'a, 'ra, 'tcx> Visitor<'a> for UnusedImportCheckVisitor<'a, 'ra, 'tcx> {
             _ => {}
         }
 
-        self.item_span = item.span_with_attributes();
         visit::walk_item(self, item);
     }
 
-    fn visit_use_tree(&mut self, use_tree: &'a ast::UseTree, id: ast::NodeId, nested: bool) {
-        // Use the base UseTree's NodeId as the item id
-        // This allows the grouping of all the lints in the same item
-        if !nested {
-            self.base_id = id;
-            self.base_use_tree = Some(use_tree);
-        }
-
-        if self.r.effective_visibilities.is_exported(self.r.local_def_id(id)) {
-            self.check_import_as_underscore(use_tree, id);
-            return;
-        }
-
-        if let ast::UseTreeKind::Nested { ref items, .. } = use_tree.kind {
-            if items.is_empty() {
-                self.unused_import(self.base_id).add(id);
-            }
-        } else {
-            self.check_import(id);
-        }
-
-        visit::walk_use_tree(self, use_tree, id);
+    fn visit_nested_use_tree(&mut self, use_tree: &'a ast::UseTree, id: ast::NodeId) {
+        self.check_use_tree(use_tree, id);
+        visit::walk_use_tree(self, use_tree);
     }
 }
 
diff --git a/compiler/rustc_resolve/src/def_collector.rs b/compiler/rustc_resolve/src/def_collector.rs
index 13dfb59f27f..dc16fe212b1 100644
--- a/compiler/rustc_resolve/src/def_collector.rs
+++ b/compiler/rustc_resolve/src/def_collector.rs
@@ -147,7 +147,10 @@ impl<'a, 'ra, 'tcx> visit::Visitor<'a> for DefCollector<'a, 'ra, 'tcx> {
                 DefKind::Macro(macro_kind)
             }
             ItemKind::GlobalAsm(..) => DefKind::GlobalAsm,
-            ItemKind::Use(..) => return visit::walk_item(self, i),
+            ItemKind::Use(use_tree) => {
+                self.create_def(i.id, None, DefKind::Use, use_tree.span);
+                return visit::walk_item(self, i);
+            }
             ItemKind::MacCall(..) | ItemKind::DelegationMac(..) => {
                 return self.visit_macro_invoc(i.id);
             }
@@ -162,8 +165,8 @@ impl<'a, 'ra, 'tcx> visit::Visitor<'a> for DefCollector<'a, 'ra, 'tcx> {
         self.with_parent(def_id, |this| {
             this.with_impl_trait(ImplTraitContext::Existential, |this| {
                 match i.kind {
-                    ItemKind::Struct(_, ref struct_def, _)
-                    | ItemKind::Union(_, ref struct_def, _) => {
+                    ItemKind::Struct(_, _, ref struct_def)
+                    | ItemKind::Union(_, _, ref struct_def) => {
                         // If this is a unit or tuple-like struct, register the constructor.
                         if let Some((ctor_kind, ctor_node_id)) = CtorKind::from_ast(struct_def) {
                             this.create_def(
@@ -232,9 +235,9 @@ impl<'a, 'ra, 'tcx> visit::Visitor<'a> for DefCollector<'a, 'ra, 'tcx> {
         }
     }
 
-    fn visit_use_tree(&mut self, use_tree: &'a UseTree, id: NodeId, _nested: bool) {
+    fn visit_nested_use_tree(&mut self, use_tree: &'a UseTree, id: NodeId) {
         self.create_def(id, None, DefKind::Use, use_tree.span);
-        visit::walk_use_tree(self, use_tree, id);
+        visit::walk_use_tree(self, use_tree);
     }
 
     fn visit_foreign_item(&mut self, fi: &'a ForeignItem) {
diff --git a/compiler/rustc_resolve/src/diagnostics.rs b/compiler/rustc_resolve/src/diagnostics.rs
index d09750fa281..201b1c0a493 100644
--- a/compiler/rustc_resolve/src/diagnostics.rs
+++ b/compiler/rustc_resolve/src/diagnostics.rs
@@ -6,7 +6,7 @@ use rustc_ast::{
 };
 use rustc_ast_pretty::pprust;
 use rustc_attr_data_structures::{self as attr, Stability};
-use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::unord::UnordSet;
 use rustc_errors::codes::*;
 use rustc_errors::{
@@ -2623,7 +2623,53 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
         };
 
         for &StrippedCfgItem { parent_module, ident, ref cfg } in symbols {
-            if parent_module != module || ident.name != *segment {
+            if ident.name != *segment {
+                continue;
+            }
+
+            fn comes_from_same_module_for_glob(
+                r: &Resolver<'_, '_>,
+                parent_module: DefId,
+                module: DefId,
+                visited: &mut FxHashMap<DefId, bool>,
+            ) -> bool {
+                if let Some(&cached) = visited.get(&parent_module) {
+                    // this branch is prevent from being called recursively infinity,
+                    // because there has some cycles in globs imports,
+                    // see more spec case at `tests/ui/cfg/diagnostics-reexport-2.rs#reexport32`
+                    return cached;
+                }
+                visited.insert(parent_module, false);
+                let res = r.module_map.get(&parent_module).is_some_and(|m| {
+                    for importer in m.glob_importers.borrow().iter() {
+                        if let Some(next_parent_module) = importer.parent_scope.module.opt_def_id()
+                        {
+                            if next_parent_module == module
+                                || comes_from_same_module_for_glob(
+                                    r,
+                                    next_parent_module,
+                                    module,
+                                    visited,
+                                )
+                            {
+                                return true;
+                            }
+                        }
+                    }
+                    false
+                });
+                visited.insert(parent_module, res);
+                res
+            }
+
+            let comes_from_same_module = parent_module == module
+                || comes_from_same_module_for_glob(
+                    self,
+                    parent_module,
+                    module,
+                    &mut Default::default(),
+                );
+            if !comes_from_same_module {
                 continue;
             }
 
diff --git a/compiler/rustc_resolve/src/effective_visibilities.rs b/compiler/rustc_resolve/src/effective_visibilities.rs
index a5ca4565d7b..5de80de3f8d 100644
--- a/compiler/rustc_resolve/src/effective_visibilities.rs
+++ b/compiler/rustc_resolve/src/effective_visibilities.rs
@@ -252,7 +252,7 @@ impl<'a, 'ra, 'tcx> Visitor<'a> for EffectiveVisibilitiesVisitor<'a, 'ra, 'tcx>
                 self.current_private_vis = prev_private_vis;
             }
 
-            ast::ItemKind::Enum(_, EnumDef { ref variants }, _) => {
+            ast::ItemKind::Enum(_, _, EnumDef { ref variants }) => {
                 self.set_bindings_effective_visibilities(def_id);
                 for variant in variants {
                     let variant_def_id = self.r.local_def_id(variant.id);
@@ -262,7 +262,7 @@ impl<'a, 'ra, 'tcx> Visitor<'a> for EffectiveVisibilitiesVisitor<'a, 'ra, 'tcx>
                 }
             }
 
-            ast::ItemKind::Struct(_, ref def, _) | ast::ItemKind::Union(_, ref def, _) => {
+            ast::ItemKind::Struct(_, _, ref def) | ast::ItemKind::Union(_, _, ref def) => {
                 for field in def.fields() {
                     self.update_field(self.r.local_def_id(field.id), def_id);
                 }
diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs
index 1b682d0cf8a..3dc285fdab6 100644
--- a/compiler/rustc_resolve/src/late.rs
+++ b/compiler/rustc_resolve/src/late.rs
@@ -415,7 +415,7 @@ pub(crate) enum AliasPossibility {
 }
 
 #[derive(Copy, Clone, Debug)]
-pub(crate) enum PathSource<'a> {
+pub(crate) enum PathSource<'a, 'c> {
     /// Type paths `Path`.
     Type,
     /// Trait paths in bounds or impls.
@@ -429,7 +429,10 @@ pub(crate) enum PathSource<'a> {
     /// Paths in tuple struct patterns `Path(..)`.
     TupleStruct(Span, &'a [Span]),
     /// `m::A::B` in `<T as m::A>::B::C`.
-    TraitItem(Namespace),
+    ///
+    /// Second field holds the "cause" of this one, i.e. the context within
+    /// which the trait item is resolved. Used for diagnostics.
+    TraitItem(Namespace, &'c PathSource<'a, 'c>),
     /// Paths in delegation item
     Delegation,
     /// An arg in a `use<'a, N>` precise-capturing bound.
@@ -440,7 +443,7 @@ pub(crate) enum PathSource<'a> {
     DefineOpaques,
 }
 
-impl<'a> PathSource<'a> {
+impl<'a> PathSource<'a, '_> {
     fn namespace(self) -> Namespace {
         match self {
             PathSource::Type
@@ -452,7 +455,7 @@ impl<'a> PathSource<'a> {
             | PathSource::TupleStruct(..)
             | PathSource::Delegation
             | PathSource::ReturnTypeNotation => ValueNS,
-            PathSource::TraitItem(ns) => ns,
+            PathSource::TraitItem(ns, _) => ns,
             PathSource::PreciseCapturingArg(ns) => ns,
         }
     }
@@ -480,8 +483,9 @@ impl<'a> PathSource<'a> {
             PathSource::Trait(_) => "trait",
             PathSource::Pat => "unit struct, unit variant or constant",
             PathSource::Struct => "struct, variant or union type",
-            PathSource::TupleStruct(..) => "tuple struct or tuple variant",
-            PathSource::TraitItem(ns) => match ns {
+            PathSource::TraitItem(ValueNS, PathSource::TupleStruct(..))
+            | PathSource::TupleStruct(..) => "tuple struct or tuple variant",
+            PathSource::TraitItem(ns, _) => match ns {
                 TypeNS => "associated type",
                 ValueNS => "method or associated constant",
                 MacroNS => bug!("associated macro"),
@@ -585,7 +589,7 @@ impl<'a> PathSource<'a> {
                 ) | Res::SelfTyParam { .. }
                     | Res::SelfTyAlias { .. }
             ),
-            PathSource::TraitItem(ns) => match res {
+            PathSource::TraitItem(ns, _) => match res {
                 Res::Def(DefKind::AssocConst | DefKind::AssocFn, _) if ns == ValueNS => true,
                 Res::Def(DefKind::AssocTy, _) if ns == TypeNS => true,
                 _ => false,
@@ -870,7 +874,7 @@ impl<'ra: 'ast, 'ast, 'tcx> Visitor<'ast> for LateResolutionVisitor<'_, 'ast, 'r
                             kind: LifetimeBinderKind::PolyTrait,
                             span,
                         },
-                        |this| this.visit_path(path, ty.id),
+                        |this| this.visit_path(path),
                     );
                 } else {
                     visit::walk_ty(self, ty)
@@ -934,8 +938,7 @@ impl<'ra: 'ast, 'ast, 'tcx> Visitor<'ast> for LateResolutionVisitor<'_, 'ast, 'r
                 )
             }
             TyKind::UnsafeBinder(unsafe_binder) => {
-                // FIXME(unsafe_binder): Better span
-                let span = ty.span;
+                let span = ty.span.shrink_to_lo().to(unsafe_binder.inner_ty.span.shrink_to_lo());
                 self.with_generic_param_rib(
                     &unsafe_binder.generic_params,
                     RibKind::Normal,
@@ -1262,7 +1265,7 @@ impl<'ra: 'ast, 'ast, 'tcx> Visitor<'ast> for LateResolutionVisitor<'_, 'ast, 'r
                             AnonConstKind::ConstArg(IsRepeatExpr::No),
                             |this| {
                                 this.smart_resolve_path(ty.id, &None, path, PathSource::Expr(None));
-                                this.visit_path(path, ty.id);
+                                this.visit_path(path);
                             },
                         );
 
@@ -1726,7 +1729,7 @@ impl<'a, 'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
         if ident.name == kw::StaticLifetime {
             self.record_lifetime_res(
                 lifetime.id,
-                LifetimeRes::Static { suppress_elision_warning: false },
+                LifetimeRes::Static,
                 LifetimeElisionCandidate::Named,
             );
             return;
@@ -1874,8 +1877,7 @@ impl<'a, 'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
                     if lifetimes_in_scope.is_empty() {
                         self.record_lifetime_res(
                             lifetime.id,
-                            // We are inside a const item, so do not warn.
-                            LifetimeRes::Static { suppress_elision_warning: true },
+                            LifetimeRes::Static,
                             elision_candidate,
                         );
                         return;
@@ -2008,7 +2010,7 @@ impl<'a, 'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
         &mut self,
         partial_res: PartialRes,
         path: &[Segment],
-        source: PathSource<'_>,
+        source: PathSource<'_, '_>,
         path_span: Span,
     ) {
         let proj_start = path.len() - partial_res.unresolved_segments();
@@ -2222,47 +2224,6 @@ impl<'a, 'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
             panic!("lifetime {id:?} resolved multiple times ({prev_res:?} before, {res:?} now)")
         }
 
-        match candidate {
-            LifetimeElisionCandidate::Missing(missing @ MissingLifetime { .. }) => {
-                debug_assert_eq!(id, missing.id);
-                match res {
-                    LifetimeRes::Static { suppress_elision_warning } => {
-                        if !suppress_elision_warning {
-                            self.r.lint_buffer.buffer_lint(
-                                lint::builtin::ELIDED_NAMED_LIFETIMES,
-                                missing.id_for_lint,
-                                missing.span,
-                                BuiltinLintDiag::ElidedNamedLifetimes {
-                                    elided: (missing.span, missing.kind),
-                                    resolution: lint::ElidedLifetimeResolution::Static,
-                                },
-                            );
-                        }
-                    }
-                    LifetimeRes::Param { param, binder: _ } => {
-                        let tcx = self.r.tcx();
-                        self.r.lint_buffer.buffer_lint(
-                            lint::builtin::ELIDED_NAMED_LIFETIMES,
-                            missing.id_for_lint,
-                            missing.span,
-                            BuiltinLintDiag::ElidedNamedLifetimes {
-                                elided: (missing.span, missing.kind),
-                                resolution: lint::ElidedLifetimeResolution::Param(
-                                    tcx.item_name(param.into()),
-                                    tcx.source_span(param),
-                                ),
-                            },
-                        );
-                    }
-                    LifetimeRes::Fresh { .. }
-                    | LifetimeRes::Infer
-                    | LifetimeRes::Error
-                    | LifetimeRes::ElidedAnchor { .. } => {}
-                }
-            }
-            LifetimeElisionCandidate::Ignore | LifetimeElisionCandidate::Named => {}
-        }
-
         match res {
             LifetimeRes::Param { .. } | LifetimeRes::Fresh { .. } | LifetimeRes::Static { .. } => {
                 if let Some(ref mut candidates) = self.lifetime_elision_candidates {
@@ -2694,9 +2655,9 @@ impl<'a, 'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
                 self.resolve_define_opaques(define_opaque);
             }
 
-            ItemKind::Enum(_, _, ref generics)
-            | ItemKind::Struct(_, _, ref generics)
-            | ItemKind::Union(_, _, ref generics) => {
+            ItemKind::Enum(_, ref generics, _)
+            | ItemKind::Struct(_, ref generics, _)
+            | ItemKind::Union(_, ref generics, _) => {
                 self.resolve_adt(item, generics);
             }
 
@@ -2785,14 +2746,9 @@ impl<'a, 'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
                 ..
             }) => {
                 self.with_static_rib(def_kind, |this| {
-                    this.with_lifetime_rib(
-                        LifetimeRibKind::Elided(LifetimeRes::Static {
-                            suppress_elision_warning: true,
-                        }),
-                        |this| {
-                            this.visit_ty(ty);
-                        },
-                    );
+                    this.with_lifetime_rib(LifetimeRibKind::Elided(LifetimeRes::Static), |this| {
+                        this.visit_ty(ty);
+                    });
                     if let Some(expr) = expr {
                         // We already forbid generic params because of the above item rib,
                         // so it doesn't matter whether this is a trivial constant.
@@ -2829,9 +2785,7 @@ impl<'a, 'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
                         this.visit_generics(generics);
 
                         this.with_lifetime_rib(
-                            LifetimeRibKind::Elided(LifetimeRes::Static {
-                                suppress_elision_warning: true,
-                            }),
+                            LifetimeRibKind::Elided(LifetimeRes::Static),
                             |this| this.visit_ty(ty),
                         );
 
@@ -3637,7 +3591,7 @@ impl<'a, 'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
         if let Some(qself) = &delegation.qself {
             self.visit_ty(&qself.ty);
         }
-        self.visit_path(&delegation.path, delegation.id);
+        self.visit_path(&delegation.path);
         let Some(body) = &delegation.body else { return };
         self.with_rib(ValueNS, RibKind::FnOrCoroutine, |this| {
             let span = delegation.path.segments.last().unwrap().ident.span;
@@ -3901,7 +3855,9 @@ impl<'a, 'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
         // We walk the pattern before declaring the pattern's inner bindings,
         // so that we avoid resolving a literal expression to a binding defined
         // by the pattern.
-        visit::walk_pat(self, pat);
+        // NB: `Self::visit_pat` must be used rather than `visit::walk_pat` to avoid resolving guard
+        // patterns' guard expressions multiple times (#141265).
+        self.visit_pat(pat);
         self.resolve_pattern_inner(pat, pat_src, bindings);
         // This has to happen *after* we determine which pat_idents are variants:
         self.check_consistent_bindings(pat);
@@ -4205,7 +4161,7 @@ impl<'a, 'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
         id: NodeId,
         qself: &Option<P<QSelf>>,
         path: &Path,
-        source: PathSource<'ast>,
+        source: PathSource<'ast, '_>,
     ) {
         self.smart_resolve_path_fragment(
             qself,
@@ -4222,7 +4178,7 @@ impl<'a, 'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
         &mut self,
         qself: &Option<P<QSelf>>,
         path: &[Segment],
-        source: PathSource<'ast>,
+        source: PathSource<'ast, '_>,
         finalize: Finalize,
         record_partial_res: RecordPartialRes,
         parent_qself: Option<&QSelf>,
@@ -4403,6 +4359,7 @@ impl<'a, 'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
             path_span,
             source.defer_to_typeck(),
             finalize,
+            source,
         ) {
             Ok(Some(partial_res)) if let Some(res) = partial_res.full_res() => {
                 // if we also have an associated type that matches the ident, stash a suggestion
@@ -4525,12 +4482,13 @@ impl<'a, 'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
         span: Span,
         defer_to_typeck: bool,
         finalize: Finalize,
+        source: PathSource<'ast, '_>,
     ) -> Result<Option<PartialRes>, Spanned<ResolutionError<'ra>>> {
         let mut fin_res = None;
 
         for (i, &ns) in [primary_ns, TypeNS, ValueNS].iter().enumerate() {
             if i == 0 || ns != primary_ns {
-                match self.resolve_qpath(qself, path, ns, finalize)? {
+                match self.resolve_qpath(qself, path, ns, finalize, source)? {
                     Some(partial_res)
                         if partial_res.unresolved_segments() == 0 || defer_to_typeck =>
                     {
@@ -4567,6 +4525,7 @@ impl<'a, 'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
         path: &[Segment],
         ns: Namespace,
         finalize: Finalize,
+        source: PathSource<'ast, '_>,
     ) -> Result<Option<PartialRes>, Spanned<ResolutionError<'ra>>> {
         debug!(
             "resolve_qpath(qself={:?}, path={:?}, ns={:?}, finalize={:?})",
@@ -4614,7 +4573,7 @@ impl<'a, 'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
             let partial_res = self.smart_resolve_path_fragment(
                 &None,
                 &path[..=qself.position],
-                PathSource::TraitItem(ns),
+                PathSource::TraitItem(ns, &source),
                 Finalize::with_root_span(finalize.node_id, finalize.path_span, qself.path_span),
                 RecordPartialRes::No,
                 Some(&qself),
@@ -4859,7 +4818,7 @@ impl<'a, 'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
                 if let Some(qself) = &se.qself {
                     self.visit_ty(&qself.ty);
                 }
-                self.visit_path(&se.path, expr.id);
+                self.visit_path(&se.path);
                 walk_list!(self, resolve_expr_field, &se.fields, expr);
                 match &se.rest {
                     StructRest::Base(expr) => self.visit_expr(expr),
@@ -4890,11 +4849,28 @@ impl<'a, 'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
                 self.resolve_expr(e, Some(expr));
             }
 
-            ExprKind::Let(ref pat, ref scrutinee, _, _) => {
+            ExprKind::Let(ref pat, ref scrutinee, _, Recovered::No) => {
                 self.visit_expr(scrutinee);
                 self.resolve_pattern_top(pat, PatternSource::Let);
             }
 
+            ExprKind::Let(ref pat, ref scrutinee, _, Recovered::Yes(_)) => {
+                self.visit_expr(scrutinee);
+                // This is basically a tweaked, inlined `resolve_pattern_top`.
+                let mut bindings = smallvec![(PatBoundCtx::Product, Default::default())];
+                self.resolve_pattern(pat, PatternSource::Let, &mut bindings);
+                // We still collect the bindings in this `let` expression which is in
+                // an invalid position (and therefore shouldn't declare variables into
+                // its parent scope). To avoid unnecessary errors though, we do just
+                // reassign the resolutions to `Res::Err`.
+                for (_, bindings) in &mut bindings {
+                    for (_, binding) in bindings {
+                        *binding = Res::Err;
+                    }
+                }
+                self.apply_pattern_bindings(bindings);
+            }
+
             ExprKind::If(ref cond, ref then, ref opt_else) => {
                 self.with_rib(ValueNS, RibKind::Normal, |this| {
                     let old = this.diag_metadata.in_if_condition.replace(cond);
@@ -5241,9 +5217,9 @@ impl<'ast> Visitor<'ast> for ItemInfoCollector<'_, '_, '_> {
             ItemKind::TyAlias(box TyAlias { generics, .. })
             | ItemKind::Const(box ConstItem { generics, .. })
             | ItemKind::Fn(box Fn { generics, .. })
-            | ItemKind::Enum(_, _, generics)
-            | ItemKind::Struct(_, _, generics)
-            | ItemKind::Union(_, _, generics)
+            | ItemKind::Enum(_, generics, _)
+            | ItemKind::Struct(_, generics, _)
+            | ItemKind::Union(_, generics, _)
             | ItemKind::Impl(box Impl { generics, .. })
             | ItemKind::Trait(box Trait { generics, .. })
             | ItemKind::TraitAlias(_, generics, _) => {
diff --git a/compiler/rustc_resolve/src/late/diagnostics.rs b/compiler/rustc_resolve/src/late/diagnostics.rs
index b538be34f31..2f6aed35f25 100644
--- a/compiler/rustc_resolve/src/late/diagnostics.rs
+++ b/compiler/rustc_resolve/src/late/diagnostics.rs
@@ -11,6 +11,7 @@ use rustc_ast::{
     Item, ItemKind, MethodCall, NodeId, Path, PathSegment, Ty, TyKind,
 };
 use rustc_ast_pretty::pprust::where_bound_predicate_to_string;
+use rustc_attr_parsing::is_doc_alias_attrs_contain_symbol;
 use rustc_data_structures::fx::{FxHashSet, FxIndexSet};
 use rustc_errors::codes::*;
 use rustc_errors::{
@@ -39,7 +40,7 @@ use crate::late::{
 };
 use crate::ty::fast_reject::SimplifiedType;
 use crate::{
-    Module, ModuleKind, ModuleOrUniformRoot, PathResult, PathSource, Segment, errors,
+    Module, ModuleKind, ModuleOrUniformRoot, PathResult, PathSource, Resolver, Segment, errors,
     path_names_to_string,
 };
 
@@ -174,7 +175,7 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
         &mut self,
         path: &[Segment],
         span: Span,
-        source: PathSource<'_>,
+        source: PathSource<'_, '_>,
         res: Option<Res>,
     ) -> BaseError {
         // Make the base error.
@@ -420,7 +421,7 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
         path: &[Segment],
         following_seg: Option<&Segment>,
         span: Span,
-        source: PathSource<'_>,
+        source: PathSource<'_, '_>,
         res: Option<Res>,
         qself: Option<&QSelf>,
     ) -> (Diag<'tcx>, Vec<ImportSuggestion>) {
@@ -477,6 +478,19 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
             return (err, Vec::new());
         }
 
+        if let Some((did, item)) = self.lookup_doc_alias_name(path, source.namespace()) {
+            let item_name = item.name;
+            let suggestion_name = self.r.tcx.item_name(did);
+            err.span_suggestion(
+                item.span,
+                format!("`{suggestion_name}` has a name defined in the doc alias attribute as `{item_name}`"),
+                    suggestion_name,
+                    Applicability::MaybeIncorrect
+                );
+
+            return (err, Vec::new());
+        };
+
         let (found, suggested_candidates, mut candidates) = self.try_lookup_name_relaxed(
             &mut err,
             source,
@@ -525,12 +539,12 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
         path: &[Segment],
         following_seg: Option<&Segment>,
         span: Span,
-        source: PathSource<'_>,
+        source: PathSource<'_, '_>,
         res: Option<Res>,
         qself: Option<&QSelf>,
     ) {
         if let Some(Res::Def(DefKind::AssocFn, _)) = res
-            && let PathSource::TraitItem(TypeNS) = source
+            && let PathSource::TraitItem(TypeNS, _) = source
             && let None = following_seg
             && let Some(qself) = qself
             && let TyKind::Path(None, ty_path) = &qself.ty.kind
@@ -636,7 +650,7 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
     fn try_lookup_name_relaxed(
         &mut self,
         err: &mut Diag<'_>,
-        source: PathSource<'_>,
+        source: PathSource<'_, '_>,
         path: &[Segment],
         following_seg: Option<&Segment>,
         span: Span,
@@ -751,12 +765,24 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
                 match candidate {
                     AssocSuggestion::Field(field_span) => {
                         if self_is_available {
-                            err.span_suggestion_verbose(
-                                span.shrink_to_lo(),
-                                "you might have meant to use the available field",
-                                format!("{pre}self."),
-                                Applicability::MachineApplicable,
-                            );
+                            let source_map = self.r.tcx.sess.source_map();
+                            // check if the field is used in a format string, such as `"{x}"`
+                            let field_is_format_named_arg = source_map
+                                .span_to_source(span, |s, start, _| {
+                                    Ok(s.get(start - 1..start) == Some("{"))
+                                });
+                            if let Ok(true) = field_is_format_named_arg {
+                                err.help(
+                                    format!("you might have meant to use the available field in a format string: `\"{{}}\", self.{}`", segment.ident.name),
+                                );
+                            } else {
+                                err.span_suggestion_verbose(
+                                    span.shrink_to_lo(),
+                                    "you might have meant to use the available field",
+                                    format!("{pre}self."),
+                                    Applicability::MaybeIncorrect,
+                                );
+                            }
                         } else {
                             err.span_label(field_span, "a field by that name exists in `Self`");
                         }
@@ -852,10 +878,69 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
         (false, suggested_candidates, candidates)
     }
 
+    fn lookup_doc_alias_name(&mut self, path: &[Segment], ns: Namespace) -> Option<(DefId, Ident)> {
+        let find_doc_alias_name = |r: &mut Resolver<'ra, '_>, m: Module<'ra>, item_name: Symbol| {
+            for resolution in r.resolutions(m).borrow().values() {
+                let Some(did) =
+                    resolution.borrow().binding.and_then(|binding| binding.res().opt_def_id())
+                else {
+                    continue;
+                };
+                if did.is_local() {
+                    // We don't record the doc alias name in the local crate
+                    // because the people who write doc alias are usually not
+                    // confused by them.
+                    continue;
+                }
+                if is_doc_alias_attrs_contain_symbol(r.tcx.get_attrs(did, sym::doc), item_name) {
+                    return Some(did);
+                }
+            }
+            None
+        };
+
+        if path.len() == 1 {
+            for rib in self.ribs[ns].iter().rev() {
+                let item = path[0].ident;
+                if let RibKind::Module(module) = rib.kind
+                    && let Some(did) = find_doc_alias_name(self.r, module, item.name)
+                {
+                    return Some((did, item));
+                }
+            }
+        } else {
+            // Finds to the last resolved module item in the path
+            // and searches doc aliases within that module.
+            //
+            // Example: For the path `a::b::last_resolved::not_exist::c::d`,
+            // we will try to find any item has doc aliases named `not_exist`
+            // in `last_resolved` module.
+            //
+            // - Use `skip(1)` because the final segment must remain unresolved.
+            for (idx, seg) in path.iter().enumerate().rev().skip(1) {
+                let Some(id) = seg.id else {
+                    continue;
+                };
+                let Some(res) = self.r.partial_res_map.get(&id) else {
+                    continue;
+                };
+                if let Res::Def(DefKind::Mod, module) = res.expect_full_res()
+                    && let Some(module) = self.r.get_module(module)
+                    && let item = path[idx + 1].ident
+                    && let Some(did) = find_doc_alias_name(self.r, module, item.name)
+                {
+                    return Some((did, item));
+                }
+                break;
+            }
+        }
+        None
+    }
+
     fn suggest_trait_and_bounds(
         &mut self,
         err: &mut Diag<'_>,
-        source: PathSource<'_>,
+        source: PathSource<'_, '_>,
         res: Option<Res>,
         span: Span,
         base_error: &BaseError,
@@ -932,7 +1017,7 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
     fn suggest_typo(
         &mut self,
         err: &mut Diag<'_>,
-        source: PathSource<'_>,
+        source: PathSource<'_, '_>,
         path: &[Segment],
         following_seg: Option<&Segment>,
         span: Span,
@@ -978,7 +1063,7 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
     fn suggest_shadowed(
         &mut self,
         err: &mut Diag<'_>,
-        source: PathSource<'_>,
+        source: PathSource<'_, '_>,
         path: &[Segment],
         following_seg: Option<&Segment>,
         span: Span,
@@ -1011,7 +1096,7 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
     fn err_code_special_cases(
         &mut self,
         err: &mut Diag<'_>,
-        source: PathSource<'_>,
+        source: PathSource<'_, '_>,
         path: &[Segment],
         span: Span,
     ) {
@@ -1056,7 +1141,7 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
     fn suggest_self_ty(
         &mut self,
         err: &mut Diag<'_>,
-        source: PathSource<'_>,
+        source: PathSource<'_, '_>,
         path: &[Segment],
         span: Span,
     ) -> bool {
@@ -1079,7 +1164,7 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
     fn suggest_self_value(
         &mut self,
         err: &mut Diag<'_>,
-        source: PathSource<'_>,
+        source: PathSource<'_, '_>,
         path: &[Segment],
         span: Span,
     ) -> bool {
@@ -1247,7 +1332,7 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
     fn suggest_swapping_misplaced_self_ty_and_trait(
         &mut self,
         err: &mut Diag<'_>,
-        source: PathSource<'_>,
+        source: PathSource<'_, '_>,
         res: Option<Res>,
         span: Span,
     ) {
@@ -1276,7 +1361,7 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
         &mut self,
         err: &mut Diag<'_>,
         res: Option<Res>,
-        source: PathSource<'_>,
+        source: PathSource<'_, '_>,
     ) {
         let PathSource::TupleStruct(_, _) = source else { return };
         let Some(Res::Def(DefKind::Fn, _)) = res else { return };
@@ -1288,7 +1373,7 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
         &mut self,
         err: &mut Diag<'_>,
         res: Option<Res>,
-        source: PathSource<'_>,
+        source: PathSource<'_, '_>,
         span: Span,
     ) {
         let PathSource::Trait(_) = source else { return };
@@ -1337,7 +1422,7 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
     fn suggest_pattern_match_with_let(
         &mut self,
         err: &mut Diag<'_>,
-        source: PathSource<'_>,
+        source: PathSource<'_, '_>,
         span: Span,
     ) -> bool {
         if let PathSource::Expr(_) = source
@@ -1363,10 +1448,10 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
     fn get_single_associated_item(
         &mut self,
         path: &[Segment],
-        source: &PathSource<'_>,
+        source: &PathSource<'_, '_>,
         filter_fn: &impl Fn(Res) -> bool,
     ) -> Option<TypoSuggestion> {
-        if let crate::PathSource::TraitItem(_) = source {
+        if let crate::PathSource::TraitItem(_, _) = source {
             let mod_path = &path[..path.len() - 1];
             if let PathResult::Module(ModuleOrUniformRoot::Module(module)) =
                 self.resolve_path(mod_path, None, None)
@@ -1471,7 +1556,7 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
 
     /// Check if the source is call expression and the first argument is `self`. If true,
     /// return the span of whole call and the span for all arguments expect the first one (`self`).
-    fn call_has_self_arg(&self, source: PathSource<'_>) -> Option<(Span, Option<Span>)> {
+    fn call_has_self_arg(&self, source: PathSource<'_, '_>) -> Option<(Span, Option<Span>)> {
         let mut has_self_arg = None;
         if let PathSource::Expr(Some(parent)) = source
             && let ExprKind::Call(_, args) = &parent.kind
@@ -1529,7 +1614,7 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
         &mut self,
         err: &mut Diag<'_>,
         span: Span,
-        source: PathSource<'_>,
+        source: PathSource<'_, '_>,
         path: &[Segment],
         res: Res,
         path_str: &str,
@@ -1581,7 +1666,7 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
             }
         };
 
-        let find_span = |source: &PathSource<'_>, err: &mut Diag<'_>| {
+        let find_span = |source: &PathSource<'_, '_>, err: &mut Diag<'_>| {
             match source {
                 PathSource::Expr(Some(Expr { span, kind: ExprKind::Call(_, _), .. }))
                 | PathSource::TupleStruct(span, _) => {
@@ -1965,8 +2050,86 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
                 err.span_label(span, fallback_label.to_string());
                 err.note("can't use `Self` as a constructor, you must use the implemented struct");
             }
-            (Res::Def(DefKind::TyAlias | DefKind::AssocTy, _), _) if ns == ValueNS => {
+            (
+                Res::Def(DefKind::TyAlias | DefKind::AssocTy, _),
+                PathSource::TraitItem(ValueNS, PathSource::TupleStruct(whole, args)),
+            ) => {
+                err.note("can't use a type alias as tuple pattern");
+
+                let mut suggestion = Vec::new();
+
+                if let &&[first, ..] = args
+                    && let &&[.., last] = args
+                {
+                    suggestion.extend([
+                        // "0: " has to be included here so that the fix is machine applicable.
+                        //
+                        // If this would only add " { " and then the code below add "0: ",
+                        // rustfix would crash, because end of this suggestion is the same as start
+                        // of the suggestion below. Thus, we have to merge these...
+                        (span.between(first), " { 0: ".to_owned()),
+                        (last.between(whole.shrink_to_hi()), " }".to_owned()),
+                    ]);
+
+                    suggestion.extend(
+                        args.iter()
+                            .enumerate()
+                            .skip(1) // See above
+                            .map(|(index, &arg)| (arg.shrink_to_lo(), format!("{index}: "))),
+                    )
+                } else {
+                    suggestion.push((span.between(whole.shrink_to_hi()), " {}".to_owned()));
+                }
+
+                err.multipart_suggestion(
+                    "use struct pattern instead",
+                    suggestion,
+                    Applicability::MachineApplicable,
+                );
+            }
+            (
+                Res::Def(DefKind::TyAlias | DefKind::AssocTy, _),
+                PathSource::TraitItem(
+                    ValueNS,
+                    PathSource::Expr(Some(ast::Expr {
+                        span: whole,
+                        kind: ast::ExprKind::Call(_, args),
+                        ..
+                    })),
+                ),
+            ) => {
                 err.note("can't use a type alias as a constructor");
+
+                let mut suggestion = Vec::new();
+
+                if let [first, ..] = &**args
+                    && let [.., last] = &**args
+                {
+                    suggestion.extend([
+                        // "0: " has to be included here so that the fix is machine applicable.
+                        //
+                        // If this would only add " { " and then the code below add "0: ",
+                        // rustfix would crash, because end of this suggestion is the same as start
+                        // of the suggestion below. Thus, we have to merge these...
+                        (span.between(first.span), " { 0: ".to_owned()),
+                        (last.span.between(whole.shrink_to_hi()), " }".to_owned()),
+                    ]);
+
+                    suggestion.extend(
+                        args.iter()
+                            .enumerate()
+                            .skip(1) // See above
+                            .map(|(index, arg)| (arg.span.shrink_to_lo(), format!("{index}: "))),
+                    )
+                } else {
+                    suggestion.push((span.between(whole.shrink_to_hi()), " {}".to_owned()));
+                }
+
+                err.multipart_suggestion(
+                    "use struct expression instead",
+                    suggestion,
+                    Applicability::MachineApplicable,
+                );
             }
             _ => return false,
         }
@@ -2536,7 +2699,7 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
     fn suggest_using_enum_variant(
         &mut self,
         err: &mut Diag<'_>,
-        source: PathSource<'_>,
+        source: PathSource<'_, '_>,
         def_id: DefId,
         span: Span,
     ) {
@@ -2714,7 +2877,7 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
     pub(crate) fn suggest_adding_generic_parameter(
         &self,
         path: &[Segment],
-        source: PathSource<'_>,
+        source: PathSource<'_, '_>,
     ) -> Option<(Span, &'static str, String, Applicability)> {
         let (ident, span) = match path {
             [segment]
@@ -3277,7 +3440,7 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
                     maybe_static = true;
                     in_scope_lifetimes = vec![(
                         Ident::with_dummy_span(kw::StaticLifetime),
-                        (DUMMY_NODE_ID, LifetimeRes::Static { suppress_elision_warning: false }),
+                        (DUMMY_NODE_ID, LifetimeRes::Static),
                     )];
                 }
             } else if elided_len == 0 {
@@ -3289,7 +3452,7 @@ impl<'ast, 'ra: 'ast, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
                     maybe_static = true;
                     in_scope_lifetimes = vec![(
                         Ident::with_dummy_span(kw::StaticLifetime),
-                        (DUMMY_NODE_ID, LifetimeRes::Static { suppress_elision_warning: false }),
+                        (DUMMY_NODE_ID, LifetimeRes::Static),
                     )];
                 }
             } else if num_params == 1 {
diff --git a/compiler/rustc_resolve/src/rustdoc.rs b/compiler/rustc_resolve/src/rustdoc.rs
index a32fe699016..fa839d2748d 100644
--- a/compiler/rustc_resolve/src/rustdoc.rs
+++ b/compiler/rustc_resolve/src/rustdoc.rs
@@ -12,10 +12,14 @@ use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::unord::UnordSet;
 use rustc_middle::ty::TyCtxt;
 use rustc_span::def_id::DefId;
-use rustc_span::{DUMMY_SP, InnerSpan, Span, Symbol, kw, sym};
+use rustc_span::source_map::SourceMap;
+use rustc_span::{DUMMY_SP, InnerSpan, Span, Symbol, sym};
 use thin_vec::ThinVec;
 use tracing::{debug, trace};
 
+#[cfg(test)]
+mod tests;
+
 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
 pub enum DocFragmentKind {
     /// A doc fragment created from a `///` or `//!` doc comment.
@@ -157,7 +161,7 @@ pub fn unindent_doc_fragments(docs: &mut [DocFragment]) {
     };
 
     for fragment in docs {
-        if fragment.doc == kw::Empty {
+        if fragment.doc == sym::empty {
             continue;
         }
 
@@ -177,7 +181,7 @@ pub fn unindent_doc_fragments(docs: &mut [DocFragment]) {
 ///
 /// Note: remove the trailing newline where appropriate
 pub fn add_doc_fragment(out: &mut String, frag: &DocFragment) {
-    if frag.doc == kw::Empty {
+    if frag.doc == sym::empty {
         out.push('\n');
         return;
     }
@@ -514,20 +518,40 @@ pub fn span_of_fragments(fragments: &[DocFragment]) -> Option<Span> {
 /// This method does not always work, because markdown bytes don't necessarily match source bytes,
 /// like if escapes are used in the string. In this case, it returns `None`.
 ///
-/// This method will return `Some` only if:
+/// `markdown` is typically the entire documentation for an item,
+/// after combining fragments.
+///
+/// This method will return `Some` only if one of the following is true:
 ///
 /// - The doc is made entirely from sugared doc comments, which cannot contain escapes
-/// - The doc is entirely from a single doc fragment, with a string literal, exactly equal
+/// - The doc is entirely from a single doc fragment with a string literal exactly equal to `markdown`.
 /// - The doc comes from `include_str!`
+/// - The doc includes exactly one substring matching `markdown[md_range]` which is contained in a single doc fragment.
+///
+/// This function is defined in the compiler so it can be used by
+/// both `rustdoc` and `clippy`.
 pub fn source_span_for_markdown_range(
     tcx: TyCtxt<'_>,
     markdown: &str,
     md_range: &Range<usize>,
     fragments: &[DocFragment],
 ) -> Option<Span> {
+    let map = tcx.sess.source_map();
+    source_span_for_markdown_range_inner(map, markdown, md_range, fragments)
+}
+
+// inner function used for unit testing
+pub fn source_span_for_markdown_range_inner(
+    map: &SourceMap,
+    markdown: &str,
+    md_range: &Range<usize>,
+    fragments: &[DocFragment],
+) -> Option<Span> {
+    use rustc_span::BytePos;
+
     if let &[fragment] = &fragments
         && fragment.kind == DocFragmentKind::RawDoc
-        && let Ok(snippet) = tcx.sess.source_map().span_to_snippet(fragment.span)
+        && let Ok(snippet) = map.span_to_snippet(fragment.span)
         && snippet.trim_end() == markdown.trim_end()
         && let Ok(md_range_lo) = u32::try_from(md_range.start)
         && let Ok(md_range_hi) = u32::try_from(md_range.end)
@@ -544,10 +568,49 @@ pub fn source_span_for_markdown_range(
     let is_all_sugared_doc = fragments.iter().all(|frag| frag.kind == DocFragmentKind::SugaredDoc);
 
     if !is_all_sugared_doc {
+        // This case ignores the markdown outside of the range so that it can
+        // work in cases where the markdown is made from several different
+        // doc fragments, but the target range does not span across multiple
+        // fragments.
+        let mut match_data = None;
+        let pat = &markdown[md_range.clone()];
+        // This heirustic doesn't make sense with a zero-sized range.
+        if pat.is_empty() {
+            return None;
+        }
+        for (i, fragment) in fragments.iter().enumerate() {
+            if let Ok(snippet) = map.span_to_snippet(fragment.span)
+                && let Some(match_start) = snippet.find(pat)
+            {
+                // If there is either a match in a previous fragment, or
+                // multiple matches in this fragment, there is ambiguity.
+                // the snippet cannot be zero-sized, because it matches
+                // the pattern, which is checked to not be zero sized.
+                if match_data.is_none()
+                    && !snippet.as_bytes()[match_start + 1..]
+                        .windows(pat.len())
+                        .any(|s| s == pat.as_bytes())
+                {
+                    match_data = Some((i, match_start));
+                } else {
+                    // Heirustic produced ambiguity, return nothing.
+                    return None;
+                }
+            }
+        }
+        if let Some((i, match_start)) = match_data {
+            let sp = fragments[i].span;
+            // we need to calculate the span start,
+            // then use that in our calulations for the span end
+            let lo = sp.lo() + BytePos(match_start as u32);
+            return Some(
+                sp.with_lo(lo).with_hi(lo + BytePos((md_range.end - md_range.start) as u32)),
+            );
+        }
         return None;
     }
 
-    let snippet = tcx.sess.source_map().span_to_snippet(span_of_fragments(fragments)?).ok()?;
+    let snippet = map.span_to_snippet(span_of_fragments(fragments)?).ok()?;
 
     let starting_line = markdown[..md_range.start].matches('\n').count();
     let ending_line = starting_line + markdown[md_range.start..md_range.end].matches('\n').count();
diff --git a/compiler/rustc_resolve/src/rustdoc/tests.rs b/compiler/rustc_resolve/src/rustdoc/tests.rs
new file mode 100644
index 00000000000..221ac907e7c
--- /dev/null
+++ b/compiler/rustc_resolve/src/rustdoc/tests.rs
@@ -0,0 +1,50 @@
+use std::path::PathBuf;
+
+use rustc_span::source_map::{FilePathMapping, SourceMap};
+use rustc_span::symbol::sym;
+use rustc_span::{BytePos, Span};
+
+use super::{DocFragment, DocFragmentKind, source_span_for_markdown_range_inner};
+
+#[test]
+fn single_backtick() {
+    let sm = SourceMap::new(FilePathMapping::empty());
+    sm.new_source_file(PathBuf::from("foo.rs").into(), r#"#[doc = "`"] fn foo() {}"#.to_string());
+    let span = source_span_for_markdown_range_inner(
+        &sm,
+        "`",
+        &(0..1),
+        &[DocFragment {
+            span: Span::with_root_ctxt(BytePos(8), BytePos(11)),
+            item_id: None,
+            kind: DocFragmentKind::RawDoc,
+            doc: sym::empty, // unused placeholder
+            indent: 0,
+        }],
+    )
+    .unwrap();
+    assert_eq!(span.lo(), BytePos(9));
+    assert_eq!(span.hi(), BytePos(10));
+}
+
+#[test]
+fn utf8() {
+    // regression test for https://github.com/rust-lang/rust/issues/141665
+    let sm = SourceMap::new(FilePathMapping::empty());
+    sm.new_source_file(PathBuf::from("foo.rs").into(), r#"#[doc = "⚠"] fn foo() {}"#.to_string());
+    let span = source_span_for_markdown_range_inner(
+        &sm,
+        "⚠",
+        &(0..3),
+        &[DocFragment {
+            span: Span::with_root_ctxt(BytePos(8), BytePos(14)),
+            item_id: None,
+            kind: DocFragmentKind::RawDoc,
+            doc: sym::empty, // unused placeholder
+            indent: 0,
+        }],
+    )
+    .unwrap();
+    assert_eq!(span.lo(), BytePos(9));
+    assert_eq!(span.hi(), BytePos(12));
+}
diff --git a/compiler/rustc_sanitizers/src/cfi/typeid/itanium_cxx_abi/encode.rs b/compiler/rustc_sanitizers/src/cfi/typeid/itanium_cxx_abi/encode.rs
index 47831f2f418..9d1d3412f8d 100644
--- a/compiler/rustc_sanitizers/src/cfi/typeid/itanium_cxx_abi/encode.rs
+++ b/compiler/rustc_sanitizers/src/cfi/typeid/itanium_cxx_abi/encode.rs
@@ -79,7 +79,7 @@ fn encode_args<'tcx>(
         s.push('I');
         let def_generics = tcx.generics_of(for_def);
         for (n, arg) in args.iter().enumerate() {
-            match arg.unpack() {
+            match arg.kind() {
                 GenericArgKind::Lifetime(region) => {
                     s.push_str(&encode_region(region, dict));
                 }
@@ -245,7 +245,7 @@ fn encode_predicate<'tcx>(
             let name = encode_ty_name(tcx, projection.def_id);
             let _ = write!(s, "u{}{}", name.len(), name);
             s.push_str(&encode_args(tcx, projection.args, projection.def_id, true, dict, options));
-            match projection.term.unpack() {
+            match projection.term.kind() {
                 TermKind::Ty(ty) => s.push_str(&encode_ty(tcx, ty, dict, options)),
                 TermKind::Const(c) => s.push_str(&encode_const(
                     tcx,
diff --git a/compiler/rustc_sanitizers/src/cfi/typeid/itanium_cxx_abi/mod.rs b/compiler/rustc_sanitizers/src/cfi/typeid/itanium_cxx_abi/mod.rs
index 562e288afaa..82e18ad497b 100644
--- a/compiler/rustc_sanitizers/src/cfi/typeid/itanium_cxx_abi/mod.rs
+++ b/compiler/rustc_sanitizers/src/cfi/typeid/itanium_cxx_abi/mod.rs
@@ -4,10 +4,11 @@
 //! For more information about LLVM CFI and cross-language LLVM CFI support for the Rust compiler,
 //! see design document in the tracking issue #89653.
 
+use rustc_abi::CanonAbi;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_middle::bug;
 use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypeFoldable, TypeVisitableExt};
-use rustc_target::callconv::{Conv, FnAbi, PassMode};
+use rustc_target::callconv::{FnAbi, PassMode};
 use tracing::instrument;
 
 mod encode;
@@ -45,7 +46,7 @@ pub fn typeid_for_fnabi<'tcx>(
     let mut encode_ty_options = EncodeTyOptions::from_bits(options.bits())
         .unwrap_or_else(|| bug!("typeid_for_fnabi: invalid option(s) `{:?}`", options.bits()));
     match fn_abi.conv {
-        Conv::C => {
+        CanonAbi::C => {
             encode_ty_options.insert(EncodeTyOptions::GENERALIZE_REPR_C);
         }
         _ => {
diff --git a/compiler/rustc_sanitizers/src/cfi/typeid/itanium_cxx_abi/transform.rs b/compiler/rustc_sanitizers/src/cfi/typeid/itanium_cxx_abi/transform.rs
index d2917478e4e..c69991f3fb2 100644
--- a/compiler/rustc_sanitizers/src/cfi/typeid/itanium_cxx_abi/transform.rs
+++ b/compiler/rustc_sanitizers/src/cfi/typeid/itanium_cxx_abi/transform.rs
@@ -14,7 +14,7 @@ use rustc_middle::ty::{
     TyCtxt, TypeFoldable, TypeFolder, TypeSuperFoldable, TypeVisitableExt, UintTy,
 };
 use rustc_span::def_id::DefId;
-use rustc_span::sym;
+use rustc_span::{DUMMY_SP, sym};
 use rustc_trait_selection::traits;
 use tracing::{debug, instrument};
 
@@ -414,7 +414,7 @@ pub(crate) fn transform_instance<'tcx>(
                 }
                 ty::Coroutine(..) => match tcx.coroutine_kind(instance.def_id()).unwrap() {
                     hir::CoroutineKind::Coroutine(..) => (
-                        tcx.require_lang_item(LangItem::Coroutine, None),
+                        tcx.require_lang_item(LangItem::Coroutine, DUMMY_SP),
                         Some(instance.args.as_coroutine().resume_ty()),
                     ),
                     hir::CoroutineKind::Desugared(desugaring, _) => {
@@ -423,11 +423,11 @@ pub(crate) fn transform_instance<'tcx>(
                             hir::CoroutineDesugaring::AsyncGen => LangItem::AsyncIterator,
                             hir::CoroutineDesugaring::Gen => LangItem::Iterator,
                         };
-                        (tcx.require_lang_item(lang_item, None), None)
+                        (tcx.require_lang_item(lang_item, DUMMY_SP), None)
                     }
                 },
                 ty::CoroutineClosure(..) => (
-                    tcx.require_lang_item(LangItem::FnOnce, None),
+                    tcx.require_lang_item(LangItem::FnOnce, DUMMY_SP),
                     Some(
                         tcx.instantiate_bound_regions_with_erased(
                             instance.args.as_coroutine_closure().coroutine_closure_sig(),
diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs
index 144aeb5c369..60e1b465ba9 100644
--- a/compiler/rustc_session/src/config.rs
+++ b/compiler/rustc_session/src/config.rs
@@ -195,11 +195,6 @@ pub struct CoverageOptions {
     /// regression tests for #133606, because we don't have an easy way to
     /// reproduce it from actual source code.
     pub discard_all_spans_in_codegen: bool,
-
-    /// `-Zcoverage-options=inject-unused-local-file`: During codegen, add an
-    /// extra dummy entry to each function's local file table, to exercise the
-    /// code that checks for local file IDs with no mapping regions.
-    pub inject_unused_local_file: bool,
 }
 
 /// Controls whether branch coverage or MC/DC coverage is enabled.
diff --git a/compiler/rustc_session/src/filesearch.rs b/compiler/rustc_session/src/filesearch.rs
index 92f1bd8ab73..def2cc97f06 100644
--- a/compiler/rustc_session/src/filesearch.rs
+++ b/compiler/rustc_session/src/filesearch.rs
@@ -3,7 +3,7 @@
 use std::path::{Path, PathBuf};
 use std::{env, fs};
 
-use rustc_fs_util::{fix_windows_verbatim_for_gcc, try_canonicalize};
+use rustc_fs_util::try_canonicalize;
 use rustc_target::spec::Target;
 use smallvec::{SmallVec, smallvec};
 
@@ -87,7 +87,7 @@ fn current_dll_path() -> Result<PathBuf, String> {
                 };
                 let bytes = CStr::from_ptr(fname_ptr).to_bytes();
                 let os = OsStr::from_bytes(bytes);
-                Ok(PathBuf::from(os))
+                try_canonicalize(Path::new(os)).map_err(|e| e.to_string())
             }
 
             #[cfg(target_os = "aix")]
@@ -122,7 +122,7 @@ fn current_dll_path() -> Result<PathBuf, String> {
                     if (data_base..data_end).contains(&addr) {
                         let bytes = CStr::from_ptr(&(*current).ldinfo_filename[0]).to_bytes();
                         let os = OsStr::from_bytes(bytes);
-                        return Ok(PathBuf::from(os));
+                        return try_canonicalize(Path::new(os)).map_err(|e| e.to_string());
                     }
                     if (*current).ldinfo_next == 0 {
                         break;
@@ -169,40 +169,26 @@ fn current_dll_path() -> Result<PathBuf, String> {
 
     filename.truncate(n);
 
-    Ok(OsString::from_wide(&filename).into())
+    let path = try_canonicalize(OsString::from_wide(&filename)).map_err(|e| e.to_string())?;
+
+    // See comments on this target function, but the gist is that
+    // gcc chokes on verbatim paths which fs::canonicalize generates
+    // so we try to avoid those kinds of paths.
+    Ok(rustc_fs_util::fix_windows_verbatim_for_gcc(&path))
 }
 
-pub fn sysroot_candidates() -> SmallVec<[PathBuf; 2]> {
-    let target = crate::config::host_tuple();
-    let mut sysroot_candidates: SmallVec<[PathBuf; 2]> = smallvec![get_or_default_sysroot()];
-    let path = current_dll_path().and_then(|s| try_canonicalize(s).map_err(|e| e.to_string()));
-    if let Ok(dll) = path {
-        // use `parent` twice to chop off the file name and then also the
-        // directory containing the dll which should be either `lib` or `bin`.
-        if let Some(path) = dll.parent().and_then(|p| p.parent()) {
-            // The original `path` pointed at the `rustc_driver` crate's dll.
-            // Now that dll should only be in one of two locations. The first is
-            // in the compiler's libdir, for example `$sysroot/lib/*.dll`. The
-            // other is the target's libdir, for example
-            // `$sysroot/lib/rustlib/$target/lib/*.dll`.
-            //
-            // We don't know which, so let's assume that if our `path` above
-            // ends in `$target` we *could* be in the target libdir, and always
-            // assume that we may be in the main libdir.
-            sysroot_candidates.push(path.to_owned());
-
-            if path.ends_with(target) {
-                sysroot_candidates.extend(
-                    path.parent() // chop off `$target`
-                        .and_then(|p| p.parent()) // chop off `rustlib`
-                        .and_then(|p| p.parent()) // chop off `lib`
-                        .map(|s| s.to_owned()),
-                );
-            }
-        }
-    }
+#[cfg(target_os = "wasi")]
+fn current_dll_path() -> Result<PathBuf, String> {
+    Err("current_dll_path is not supported on WASI".to_string())
+}
 
-    sysroot_candidates
+pub fn sysroot_with_fallback(sysroot: &Path) -> SmallVec<[PathBuf; 2]> {
+    let mut candidates = smallvec![sysroot.to_owned()];
+    let default_sysroot = get_or_default_sysroot();
+    if default_sysroot != sysroot {
+        candidates.push(default_sysroot);
+    }
+    candidates
 }
 
 /// Returns the provided sysroot or calls [`get_or_default_sysroot`] if it's none.
@@ -214,17 +200,8 @@ pub fn materialize_sysroot(maybe_sysroot: Option<PathBuf>) -> PathBuf {
 /// This function checks if sysroot is found using env::args().next(), and if it
 /// is not found, finds sysroot from current rustc_driver dll.
 pub fn get_or_default_sysroot() -> PathBuf {
-    // Follow symlinks. If the resolved path is relative, make it absolute.
-    fn canonicalize(path: PathBuf) -> PathBuf {
-        let path = try_canonicalize(&path).unwrap_or(path);
-        // See comments on this target function, but the gist is that
-        // gcc chokes on verbatim paths which fs::canonicalize generates
-        // so we try to avoid those kinds of paths.
-        fix_windows_verbatim_for_gcc(&path)
-    }
-
     fn default_from_rustc_driver_dll() -> Result<PathBuf, String> {
-        let dll = current_dll_path().map(|s| canonicalize(s))?;
+        let dll = current_dll_path()?;
 
         // `dll` will be in one of the following two:
         // - compiler's libdir: $sysroot/lib/*.dll
@@ -237,7 +214,7 @@ pub fn get_or_default_sysroot() -> PathBuf {
             dll.display()
         ))?;
 
-        // if `dir` points target's dir, move up to the sysroot
+        // if `dir` points to target's dir, move up to the sysroot
         let mut sysroot_dir = if dir.ends_with(crate::config::host_tuple()) {
             dir.parent() // chop off `$target`
                 .and_then(|p| p.parent()) // chop off `rustlib`
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index 3d9fdcbc7b1..12fa05118ca 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -1413,7 +1413,6 @@ pub mod parse {
                 "mcdc" => slot.level = CoverageLevel::Mcdc,
                 "no-mir-spans" => slot.no_mir_spans = true,
                 "discard-all-spans-in-codegen" => slot.discard_all_spans_in_codegen = true,
-                "inject-unused-local-file" => slot.inject_unused_local_file = true,
                 _ => return false,
             }
         }
@@ -2367,6 +2366,8 @@ options! {
         "run LLVM in non-parallel mode (while keeping codegen-units and ThinLTO)"),
     no_profiler_runtime: bool = (false, parse_no_value, [TRACKED],
         "prevent automatic injection of the profiler_builtins crate"),
+    no_steal_thir: bool = (false, parse_bool, [UNTRACKED],
+        "don't steal the THIR when we're done with it; useful for rustc drivers (default: no)"),
     no_trait_vptr: bool = (false, parse_no_value, [TRACKED],
         "disable generation of trait vptr in vtable for upcasting"),
     no_unique_section_names: bool = (false, parse_bool, [TRACKED],
diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs
index 34ac37d6378..6b85e0abc86 100644
--- a/compiler/rustc_session/src/session.rs
+++ b/compiler/rustc_session/src/session.rs
@@ -371,11 +371,6 @@ impl Session {
         self.opts.unstable_opts.coverage_options.discard_all_spans_in_codegen
     }
 
-    /// True if testing flag `-Zcoverage-options=inject-unused-local-file` was passed.
-    pub fn coverage_inject_unused_local_file(&self) -> bool {
-        self.opts.unstable_opts.coverage_options.inject_unused_local_file
-    }
-
     pub fn is_sanitizer_cfi_enabled(&self) -> bool {
         self.opts.unstable_opts.sanitizer.contains(SanitizerSet::CFI)
     }
@@ -463,13 +458,9 @@ impl Session {
     /// directories are also returned, for example if `--sysroot` is used but tools are missing
     /// (#125246): we also add the bin directories to the sysroot where rustc is located.
     pub fn get_tools_search_paths(&self, self_contained: bool) -> Vec<PathBuf> {
-        let bin_path = filesearch::make_target_bin_path(&self.sysroot, config::host_tuple());
-        let fallback_sysroot_paths = filesearch::sysroot_candidates()
+        let search_paths = filesearch::sysroot_with_fallback(&self.sysroot)
             .into_iter()
-            // Ignore sysroot candidate if it was the same as the sysroot path we just used.
-            .filter(|sysroot| *sysroot != self.sysroot)
             .map(|sysroot| filesearch::make_target_bin_path(&sysroot, config::host_tuple()));
-        let search_paths = std::iter::once(bin_path).chain(fallback_sysroot_paths);
 
         if self_contained {
             // The self-contained tools are expected to be e.g. in `bin/self-contained` in the
diff --git a/compiler/rustc_smir/src/rustc_smir/alloc.rs b/compiler/rustc_smir/src/rustc_smir/alloc.rs
index 9cb89634c52..a6d31ac4e13 100644
--- a/compiler/rustc_smir/src/rustc_smir/alloc.rs
+++ b/compiler/rustc_smir/src/rustc_smir/alloc.rs
@@ -48,6 +48,7 @@ pub(crate) fn try_new_allocation<'tcx>(
                 size,
                 layout.align.abi,
                 AllocInit::Uninit,
+                (),
             );
             allocation
                 .write_scalar(&tables.tcx, alloc_range(Size::ZERO, size), scalar)
@@ -65,6 +66,7 @@ pub(crate) fn try_new_allocation<'tcx>(
                 layout.size,
                 layout.align.abi,
                 AllocInit::Uninit,
+                (),
             );
             allocation
                 .write_scalar(
diff --git a/compiler/rustc_smir/src/rustc_smir/convert/abi.rs b/compiler/rustc_smir/src/rustc_smir/convert/abi.rs
index 7ccc785a400..64901ee0502 100644
--- a/compiler/rustc_smir/src/rustc_smir/convert/abi.rs
+++ b/compiler/rustc_smir/src/rustc_smir/convert/abi.rs
@@ -2,8 +2,9 @@
 
 #![allow(rustc::usage_of_qualified_ty)]
 
+use rustc_abi::{ArmCall, CanonAbi, InterruptKind, X86Call};
 use rustc_middle::ty;
-use rustc_target::callconv::{self, Conv};
+use rustc_target::callconv;
 use stable_mir::abi::{
     AddressSpace, ArgAbi, CallConvention, FieldsShape, FloatLength, FnAbi, IntegerLength, Layout,
     LayoutShape, PassMode, Primitive, Scalar, TagEncoding, TyAndLayout, ValueAbi, VariantsShape,
@@ -69,7 +70,7 @@ impl<'tcx> Stable<'tcx> for callconv::FnAbi<'tcx, ty::Ty<'tcx>> {
 
     fn stable(&self, tables: &mut Tables<'_>) -> Self::T {
         assert!(self.args.len() >= self.fixed_count as usize);
-        assert!(!self.c_variadic || matches!(self.conv, Conv::C));
+        assert!(!self.c_variadic || matches!(self.conv, CanonAbi::C));
         FnAbi {
             args: self.args.as_ref().stable(tables),
             ret: self.ret.stable(tables),
@@ -92,31 +93,37 @@ impl<'tcx> Stable<'tcx> for callconv::ArgAbi<'tcx, ty::Ty<'tcx>> {
     }
 }
 
-impl<'tcx> Stable<'tcx> for callconv::Conv {
+impl<'tcx> Stable<'tcx> for CanonAbi {
     type T = CallConvention;
 
     fn stable(&self, _tables: &mut Tables<'_>) -> Self::T {
         match self {
-            Conv::C => CallConvention::C,
-            Conv::Rust => CallConvention::Rust,
-            Conv::Cold => CallConvention::Cold,
-            Conv::PreserveMost => CallConvention::PreserveMost,
-            Conv::PreserveAll => CallConvention::PreserveAll,
-            Conv::ArmAapcs => CallConvention::ArmAapcs,
-            Conv::CCmseNonSecureCall => CallConvention::CCmseNonSecureCall,
-            Conv::CCmseNonSecureEntry => CallConvention::CCmseNonSecureEntry,
-            Conv::Msp430Intr => CallConvention::Msp430Intr,
-            Conv::X86Fastcall => CallConvention::X86Fastcall,
-            Conv::X86Intr => CallConvention::X86Intr,
-            Conv::X86Stdcall => CallConvention::X86Stdcall,
-            Conv::X86ThisCall => CallConvention::X86ThisCall,
-            Conv::X86VectorCall => CallConvention::X86VectorCall,
-            Conv::X86_64SysV => CallConvention::X86_64SysV,
-            Conv::X86_64Win64 => CallConvention::X86_64Win64,
-            Conv::GpuKernel => CallConvention::GpuKernel,
-            Conv::AvrInterrupt => CallConvention::AvrInterrupt,
-            Conv::AvrNonBlockingInterrupt => CallConvention::AvrNonBlockingInterrupt,
-            Conv::RiscvInterrupt { .. } => CallConvention::RiscvInterrupt,
+            CanonAbi::C => CallConvention::C,
+            CanonAbi::Rust => CallConvention::Rust,
+            CanonAbi::RustCold => CallConvention::Cold,
+            CanonAbi::Arm(arm_call) => match arm_call {
+                ArmCall::Aapcs => CallConvention::ArmAapcs,
+                ArmCall::CCmseNonSecureCall => CallConvention::CCmseNonSecureCall,
+                ArmCall::CCmseNonSecureEntry => CallConvention::CCmseNonSecureEntry,
+            },
+            CanonAbi::GpuKernel => CallConvention::GpuKernel,
+            CanonAbi::Interrupt(interrupt_kind) => match interrupt_kind {
+                InterruptKind::Avr => CallConvention::AvrInterrupt,
+                InterruptKind::AvrNonBlocking => CallConvention::AvrNonBlockingInterrupt,
+                InterruptKind::Msp430 => CallConvention::Msp430Intr,
+                InterruptKind::RiscvMachine | InterruptKind::RiscvSupervisor => {
+                    CallConvention::RiscvInterrupt
+                }
+                InterruptKind::X86 => CallConvention::X86Intr,
+            },
+            CanonAbi::X86(x86_call) => match x86_call {
+                X86Call::Fastcall => CallConvention::X86Fastcall,
+                X86Call::Stdcall => CallConvention::X86Stdcall,
+                X86Call::SysV64 => CallConvention::X86_64SysV,
+                X86Call::Thiscall => CallConvention::X86ThisCall,
+                X86Call::Vectorcall => CallConvention::X86VectorCall,
+                X86Call::Win64 => CallConvention::X86_64Win64,
+            },
         }
     }
 }
@@ -173,7 +180,7 @@ impl<'tcx> Stable<'tcx> for rustc_abi::Variants<rustc_abi::FieldIdx, rustc_abi::
                 VariantsShape::Multiple {
                     tag: tag.stable(tables),
                     tag_encoding: tag_encoding.stable(tables),
-                    tag_field: *tag_field,
+                    tag_field: tag_field.stable(tables),
                     variants: variants.iter().as_slice().stable(tables),
                 }
             }
diff --git a/compiler/rustc_smir/src/rustc_smir/convert/ty.rs b/compiler/rustc_smir/src/rustc_smir/convert/ty.rs
index 8bcac4c4678..b0c9dba78a6 100644
--- a/compiler/rustc_smir/src/rustc_smir/convert/ty.rs
+++ b/compiler/rustc_smir/src/rustc_smir/convert/ty.rs
@@ -100,7 +100,7 @@ impl<'tcx> Stable<'tcx> for ty::ExistentialProjection<'tcx> {
         stable_mir::ty::ExistentialProjection {
             def_id: tables.trait_def(*def_id),
             generic_args: args.stable(tables),
-            term: term.unpack().stable(tables),
+            term: term.kind().stable(tables),
         }
     }
 }
@@ -158,7 +158,7 @@ impl<'tcx> Stable<'tcx> for ty::FieldDef {
 impl<'tcx> Stable<'tcx> for ty::GenericArgs<'tcx> {
     type T = stable_mir::ty::GenericArgs;
     fn stable(&self, tables: &mut Tables<'_>) -> Self::T {
-        GenericArgs(self.iter().map(|arg| arg.unpack().stable(tables)).collect())
+        GenericArgs(self.iter().map(|arg| arg.kind().stable(tables)).collect())
     }
 }
 
@@ -604,8 +604,8 @@ impl<'tcx> Stable<'tcx> for ty::PredicateKind<'tcx> {
             PredicateKind::NormalizesTo(_pred) => unimplemented!(),
             PredicateKind::AliasRelate(a, b, alias_relation_direction) => {
                 stable_mir::ty::PredicateKind::AliasRelate(
-                    a.unpack().stable(tables),
-                    b.unpack().stable(tables),
+                    a.kind().stable(tables),
+                    b.kind().stable(tables),
                     alias_relation_direction.stable(tables),
                 )
             }
@@ -640,7 +640,7 @@ impl<'tcx> Stable<'tcx> for ty::ClauseKind<'tcx> {
                 ty.stable(tables),
             ),
             ClauseKind::WellFormed(term) => {
-                stable_mir::ty::ClauseKind::WellFormed(term.unpack().stable(tables))
+                stable_mir::ty::ClauseKind::WellFormed(term.kind().stable(tables))
             }
             ClauseKind::ConstEvaluatable(const_) => {
                 stable_mir::ty::ClauseKind::ConstEvaluatable(const_.stable(tables))
@@ -726,7 +726,7 @@ impl<'tcx> Stable<'tcx> for ty::ProjectionPredicate<'tcx> {
         let ty::ProjectionPredicate { projection_term, term } = self;
         stable_mir::ty::ProjectionPredicate {
             projection_term: projection_term.stable(tables),
-            term: term.unpack().stable(tables),
+            term: term.kind().stable(tables),
         }
     }
 }
diff --git a/compiler/rustc_span/src/analyze_source_file.rs b/compiler/rustc_span/src/analyze_source_file.rs
index 6384fa06c21..55d899c9ada 100644
--- a/compiler/rustc_span/src/analyze_source_file.rs
+++ b/compiler/rustc_span/src/analyze_source_file.rs
@@ -29,7 +29,18 @@ pub(crate) fn analyze_source_file(src: &str) -> (Vec<RelativeBytePos>, Vec<Multi
     (lines, multi_byte_chars)
 }
 
-cfg_match! {
+// cfg(bootstrap)
+macro_rules! cfg_select_dispatch {
+    ($($tokens:tt)*) => {
+        #[cfg(bootstrap)]
+        cfg_match! { $($tokens)* }
+
+        #[cfg(not(bootstrap))]
+        cfg_select! { $($tokens)* }
+    };
+}
+
+cfg_select_dispatch! {
     any(target_arch = "x86", target_arch = "x86_64") => {
         fn analyze_source_file_dispatch(
             src: &str,
diff --git a/compiler/rustc_span/src/lib.rs b/compiler/rustc_span/src/lib.rs
index 22ca8accf9f..ed74dea5f1e 100644
--- a/compiler/rustc_span/src/lib.rs
+++ b/compiler/rustc_span/src/lib.rs
@@ -17,12 +17,12 @@
 
 // tidy-alphabetical-start
 #![allow(internal_features)]
+#![cfg_attr(bootstrap, feature(cfg_match))]
+#![cfg_attr(not(bootstrap), feature(cfg_select))]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(array_windows)]
-#![feature(cfg_match)]
 #![feature(core_io_borrowed_buf)]
-#![feature(hash_set_entry)]
 #![feature(if_let_guard)]
 #![feature(map_try_insert)]
 #![feature(negative_impls)]
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index efae6250b07..d66f98871b9 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -34,17 +34,8 @@ symbols! {
         // unnamed method parameters, crate root module, error recovery etc.
         // Matching predicates: `is_special`/`is_reserved`
         //
-        // Notes about `kw::Empty`:
-        // - Its use can blur the lines between "empty symbol" and "no symbol".
-        //   Using `Option<Symbol>` is preferable, where possible, because that
-        //   is unambiguous.
-        // - For dummy symbols that are never used and absolutely must be
-        //   present, it's better to use `sym::dummy` than `kw::Empty`, because
-        //   it's clearer that it's intended as a dummy value, and more likely
-        //   to be detected if it accidentally does get used.
         // tidy-alphabetical-start
         DollarCrate:        "$crate",
-        Empty:              "",
         PathRoot:           "{{root}}",
         Underscore:         "_",
         // tidy-alphabetical-end
@@ -253,6 +244,7 @@ symbols! {
         FnMut,
         FnOnce,
         Formatter,
+        Forward,
         From,
         FromIterator,
         FromResidual,
@@ -348,6 +340,7 @@ symbols! {
         Result,
         ResumeTy,
         Return,
+        Reverse,
         Right,
         Rust,
         RustaceansAreAwesome,
@@ -398,7 +391,6 @@ symbols! {
         Wrapping,
         Yield,
         _DECLS,
-        _Self,
         __D,
         __H,
         __S,
@@ -475,6 +467,7 @@ symbols! {
         as_ref,
         as_str,
         asm,
+        asm_cfg,
         asm_const,
         asm_experimental_arch,
         asm_experimental_reg,
@@ -522,7 +515,24 @@ symbols! {
         async_iterator_poll_next,
         async_trait_bounds,
         atomic,
+        atomic_and,
+        atomic_cxchg,
+        atomic_cxchgweak,
+        atomic_fence,
+        atomic_load,
+        atomic_max,
+        atomic_min,
         atomic_mod,
+        atomic_nand,
+        atomic_or,
+        atomic_singlethreadfence,
+        atomic_store,
+        atomic_umax,
+        atomic_umin,
+        atomic_xadd,
+        atomic_xchg,
+        atomic_xor,
+        atomic_xsub,
         atomics,
         att_syntax,
         attr,
@@ -531,7 +541,8 @@ symbols! {
         audit_that,
         augmented_assignments,
         auto_traits,
-        autodiff,
+        autodiff_forward,
+        autodiff_reverse,
         automatically_derived,
         avx,
         avx10_target_feature,
@@ -864,7 +875,7 @@ symbols! {
         drop_types_in_const,
         dropck_eyepatch,
         dropck_parametricity,
-        dummy: "<!dummy!>", // use this instead of `kw::Empty` for symbols that won't be used
+        dummy: "<!dummy!>", // use this instead of `sym::empty` for symbols that won't be used
         dummy_cgu_name,
         dylib,
         dyn_compatible_for_dispatch,
@@ -883,6 +894,14 @@ symbols! {
         emit_enum_variant_arg,
         emit_struct,
         emit_struct_field,
+        // Notes about `sym::empty`:
+        // - It should only be used when it genuinely means "empty symbol". Use
+        //   `Option<Symbol>` when "no symbol" is a possibility.
+        // - For dummy symbols that are never used and absolutely must be
+        //   present, it's better to use `sym::dummy` than `sym::empty`, because
+        //   it's clearer that it's intended as a dummy value, and more likely
+        //   to be detected if it accidentally does get used.
+        empty: "",
         emscripten_wasm_eh,
         enable,
         encode,
@@ -934,8 +953,10 @@ symbols! {
         external_doc,
         f,
         f128,
+        f128_epsilon,
         f128_nan,
         f16,
+        f16_epsilon,
         f16_nan,
         f16c_target_feature,
         f32,
@@ -1997,6 +2018,7 @@ symbols! {
         slice,
         slice_from_raw_parts,
         slice_from_raw_parts_mut,
+        slice_get_unchecked,
         slice_into_vec,
         slice_iter,
         slice_len_fn,
@@ -2072,6 +2094,9 @@ symbols! {
         sym,
         sync,
         synthetic,
+        sys_mutex_lock,
+        sys_mutex_try_lock,
+        sys_mutex_unlock,
         t32,
         target,
         target_abi,
@@ -2362,7 +2387,7 @@ impl Ident {
     #[inline]
     /// Constructs a new identifier from a symbol and a span.
     pub fn new(name: Symbol, span: Span) -> Ident {
-        debug_assert_ne!(name, kw::Empty);
+        debug_assert_ne!(name, sym::empty);
         Ident { name, span }
     }
 
@@ -2584,7 +2609,7 @@ impl Symbol {
     }
 
     pub fn is_empty(self) -> bool {
-        self == kw::Empty
+        self == sym::empty
     }
 
     /// This method is supposed to be used in error messages, so it's expected to be
@@ -2592,7 +2617,8 @@ impl Symbol {
     /// (`token_to_string`, `Ident::to_string`), except that symbols don't keep the rawness flag
     /// or edition, so we have to guess the rawness using the global edition.
     pub fn to_ident_string(self) -> String {
-        Ident::with_dummy_span(self).to_string()
+        // Avoid creating an empty identifier, because that asserts in debug builds.
+        if self == sym::empty { String::new() } else { Ident::with_dummy_span(self).to_string() }
     }
 }
 
@@ -2772,7 +2798,7 @@ impl Symbol {
 
     /// Returns `true` if this symbol can be a raw identifier.
     pub fn can_be_raw(self) -> bool {
-        self != kw::Empty && self != kw::Underscore && !self.is_path_segment_keyword()
+        self != sym::empty && self != kw::Underscore && !self.is_path_segment_keyword()
     }
 
     /// Was this symbol predefined in the compiler's `symbols!` macro
@@ -2822,7 +2848,7 @@ impl Ident {
     /// Whether this would be the identifier for a tuple field like `self.0`, as
     /// opposed to a named field like `self.thing`.
     pub fn is_numeric(self) -> bool {
-        !self.name.is_empty() && self.as_str().bytes().all(|b| b.is_ascii_digit())
+        self.as_str().bytes().all(|b| b.is_ascii_digit())
     }
 }
 
diff --git a/compiler/rustc_symbol_mangling/src/export.rs b/compiler/rustc_symbol_mangling/src/export.rs
index 770401fc8cf..956c996326b 100644
--- a/compiler/rustc_symbol_mangling/src/export.rs
+++ b/compiler/rustc_symbol_mangling/src/export.rs
@@ -147,7 +147,7 @@ impl<'tcx> AbiHashStable<'tcx> for ty::FnSig<'tcx> {
 
 impl<'tcx> AbiHashStable<'tcx> for ty::GenericArg<'tcx> {
     fn abi_hash(&self, tcx: TyCtxt<'tcx>, hasher: &mut StableHasher) {
-        self.unpack().abi_hash(tcx, hasher);
+        self.kind().abi_hash(tcx, hasher);
     }
 }
 
diff --git a/compiler/rustc_symbol_mangling/src/legacy.rs b/compiler/rustc_symbol_mangling/src/legacy.rs
index db102abda7f..12d1de46313 100644
--- a/compiler/rustc_symbol_mangling/src/legacy.rs
+++ b/compiler/rustc_symbol_mangling/src/legacy.rs
@@ -386,7 +386,7 @@ impl<'tcx> Printer<'tcx> for SymbolPrinter<'tcx> {
         print_prefix(self)?;
 
         let args =
-            args.iter().cloned().filter(|arg| !matches!(arg.unpack(), GenericArgKind::Lifetime(_)));
+            args.iter().cloned().filter(|arg| !matches!(arg.kind(), GenericArgKind::Lifetime(_)));
 
         if args.clone().next().is_some() {
             self.generic_delimiters(|cx| cx.comma_sep(args))
diff --git a/compiler/rustc_symbol_mangling/src/v0.rs b/compiler/rustc_symbol_mangling/src/v0.rs
index 4a99ce09b39..49a5e20d7cf 100644
--- a/compiler/rustc_symbol_mangling/src/v0.rs
+++ b/compiler/rustc_symbol_mangling/src/v0.rs
@@ -20,7 +20,7 @@ use rustc_middle::ty::{
     self, FloatTy, GenericArg, GenericArgKind, Instance, IntTy, ReifyReason, Ty, TyCtxt,
     TypeVisitable, TypeVisitableExt, UintTy,
 };
-use rustc_span::kw;
+use rustc_span::sym;
 
 pub(super) fn mangle<'tcx>(
     tcx: TyCtxt<'tcx>,
@@ -646,7 +646,7 @@ impl<'tcx> Printer<'tcx> for SymbolMangler<'tcx> {
                         let name = cx.tcx.associated_item(projection.def_id).name();
                         cx.push("p");
                         cx.push_ident(name.as_str());
-                        match projection.term.unpack() {
+                        match projection.term.kind() {
                             ty::TermKind::Ty(ty) => ty.print(cx),
                             ty::TermKind::Const(c) => c.print(cx),
                         }?;
@@ -902,7 +902,7 @@ impl<'tcx> Printer<'tcx> for SymbolMangler<'tcx> {
             print_prefix,
             ns,
             disambiguated_data.disambiguator as u64,
-            name.unwrap_or(kw::Empty).as_str(),
+            name.unwrap_or(sym::empty).as_str(),
         )
     }
 
@@ -912,11 +912,11 @@ impl<'tcx> Printer<'tcx> for SymbolMangler<'tcx> {
         args: &[GenericArg<'tcx>],
     ) -> Result<(), PrintError> {
         // Don't print any regions if they're all erased.
-        let print_regions = args.iter().any(|arg| match arg.unpack() {
+        let print_regions = args.iter().any(|arg| match arg.kind() {
             GenericArgKind::Lifetime(r) => !r.is_erased(),
             _ => false,
         });
-        let args = args.iter().cloned().filter(|arg| match arg.unpack() {
+        let args = args.iter().cloned().filter(|arg| match arg.kind() {
             GenericArgKind::Lifetime(_) => print_regions,
             _ => true,
         });
@@ -928,7 +928,7 @@ impl<'tcx> Printer<'tcx> for SymbolMangler<'tcx> {
         self.push("I");
         print_prefix(self)?;
         for arg in args {
-            match arg.unpack() {
+            match arg.kind() {
                 GenericArgKind::Lifetime(lt) => {
                     lt.print(self)?;
                 }
diff --git a/compiler/rustc_target/Cargo.toml b/compiler/rustc_target/Cargo.toml
index 189b19b0286..0121c752dbd 100644
--- a/compiler/rustc_target/Cargo.toml
+++ b/compiler/rustc_target/Cargo.toml
@@ -20,5 +20,5 @@ tracing = "0.1"
 # tidy-alphabetical-start
 default-features = false
 features = ["elf", "macho"]
-version = "0.36.2"
+version = "0.37.0"
 # tidy-alphabetical-end
diff --git a/compiler/rustc_target/src/asm/mod.rs b/compiler/rustc_target/src/asm/mod.rs
index 9f791603c72..e06f881e4b1 100644
--- a/compiler/rustc_target/src/asm/mod.rs
+++ b/compiler/rustc_target/src/asm/mod.rs
@@ -226,6 +226,7 @@ pub enum InlineAsmArch {
     RiscV64,
     Nvptx64,
     Hexagon,
+    LoongArch32,
     LoongArch64,
     Mips,
     Mips64,
@@ -260,6 +261,7 @@ impl FromStr for InlineAsmArch {
             "powerpc" => Ok(Self::PowerPC),
             "powerpc64" => Ok(Self::PowerPC64),
             "hexagon" => Ok(Self::Hexagon),
+            "loongarch32" => Ok(Self::LoongArch32),
             "loongarch64" => Ok(Self::LoongArch64),
             "mips" | "mips32r6" => Ok(Self::Mips),
             "mips64" | "mips64r6" => Ok(Self::Mips64),
@@ -365,7 +367,9 @@ impl InlineAsmReg {
                 Self::PowerPC(PowerPCInlineAsmReg::parse(name)?)
             }
             InlineAsmArch::Hexagon => Self::Hexagon(HexagonInlineAsmReg::parse(name)?),
-            InlineAsmArch::LoongArch64 => Self::LoongArch(LoongArchInlineAsmReg::parse(name)?),
+            InlineAsmArch::LoongArch32 | InlineAsmArch::LoongArch64 => {
+                Self::LoongArch(LoongArchInlineAsmReg::parse(name)?)
+            }
             InlineAsmArch::Mips | InlineAsmArch::Mips64 => {
                 Self::Mips(MipsInlineAsmReg::parse(name)?)
             }
@@ -652,7 +656,9 @@ impl InlineAsmRegClass {
                 Self::PowerPC(PowerPCInlineAsmRegClass::parse(name)?)
             }
             InlineAsmArch::Hexagon => Self::Hexagon(HexagonInlineAsmRegClass::parse(name)?),
-            InlineAsmArch::LoongArch64 => Self::LoongArch(LoongArchInlineAsmRegClass::parse(name)?),
+            InlineAsmArch::LoongArch32 | InlineAsmArch::LoongArch64 => {
+                Self::LoongArch(LoongArchInlineAsmRegClass::parse(name)?)
+            }
             InlineAsmArch::Mips | InlineAsmArch::Mips64 => {
                 Self::Mips(MipsInlineAsmRegClass::parse(name)?)
             }
@@ -860,7 +866,7 @@ pub fn allocatable_registers(
             hexagon::fill_reg_map(arch, reloc_model, target_features, target, &mut map);
             map
         }
-        InlineAsmArch::LoongArch64 => {
+        InlineAsmArch::LoongArch32 | InlineAsmArch::LoongArch64 => {
             let mut map = loongarch::regclass_map();
             loongarch::fill_reg_map(arch, reloc_model, target_features, target, &mut map);
             map
@@ -992,7 +998,7 @@ impl InlineAsmClobberAbi {
                 "C" | "system" => Ok(InlineAsmClobberAbi::Avr),
                 _ => Err(&["C", "system"]),
             },
-            InlineAsmArch::LoongArch64 => match name {
+            InlineAsmArch::LoongArch32 | InlineAsmArch::LoongArch64 => match name {
                 "C" | "system" => Ok(InlineAsmClobberAbi::LoongArch),
                 _ => Err(&["C", "system"]),
             },
diff --git a/compiler/rustc_target/src/callconv/arm.rs b/compiler/rustc_target/src/callconv/arm.rs
index 0a5dcc66347..70830fa07b6 100644
--- a/compiler/rustc_target/src/callconv/arm.rs
+++ b/compiler/rustc_target/src/callconv/arm.rs
@@ -1,6 +1,6 @@
-use rustc_abi::{HasDataLayout, TyAbiInterface};
+use rustc_abi::{ArmCall, CanonAbi, HasDataLayout, TyAbiInterface};
 
-use crate::callconv::{ArgAbi, Conv, FnAbi, Reg, RegKind, Uniform};
+use crate::callconv::{ArgAbi, FnAbi, Reg, RegKind, Uniform};
 use crate::spec::HasTargetSpec;
 
 fn is_homogeneous_aggregate<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>) -> Option<Uniform>
@@ -90,7 +90,7 @@ where
     // If this is a target with a hard-float ABI, and the function is not explicitly
     // `extern "aapcs"`, then we must use the VFP registers for homogeneous aggregates.
     let vfp = cx.target_spec().llvm_target.ends_with("hf")
-        && fn_abi.conv != Conv::ArmAapcs
+        && fn_abi.conv != CanonAbi::Arm(ArmCall::Aapcs)
         && !fn_abi.c_variadic;
 
     if !fn_abi.ret.is_ignore() {
diff --git a/compiler/rustc_target/src/callconv/mod.rs b/compiler/rustc_target/src/callconv/mod.rs
index 907614520a2..f9ecf02f857 100644
--- a/compiler/rustc_target/src/callconv/mod.rs
+++ b/compiler/rustc_target/src/callconv/mod.rs
@@ -1,14 +1,13 @@
-use std::fmt::Display;
-use std::str::FromStr;
 use std::{fmt, iter};
 
 use rustc_abi::{
-    AddressSpace, Align, BackendRepr, ExternAbi, HasDataLayout, Primitive, Reg, RegKind, Scalar,
-    Size, TyAbiInterface, TyAndLayout,
+    AddressSpace, Align, BackendRepr, CanonAbi, ExternAbi, HasDataLayout, Primitive, Reg, RegKind,
+    Scalar, Size, TyAbiInterface, TyAndLayout,
 };
 use rustc_macros::HashStable_Generic;
 
-use crate::spec::{HasTargetSpec, HasWasmCAbiOpt, HasX86AbiOpt, RustcAbi, WasmCAbi};
+pub use crate::spec::AbiMap;
+use crate::spec::{HasTargetSpec, HasWasmCAbiOpt, HasX86AbiOpt, WasmCAbi};
 
 mod aarch64;
 mod amdgpu;
@@ -530,41 +529,6 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
 }
 
 #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)]
-pub enum Conv {
-    // General language calling conventions, for which every target
-    // should have its own backend (e.g. LLVM) support.
-    C,
-    Rust,
-
-    Cold,
-    PreserveMost,
-    PreserveAll,
-
-    // Target-specific calling conventions.
-    ArmAapcs,
-    CCmseNonSecureCall,
-    CCmseNonSecureEntry,
-
-    Msp430Intr,
-
-    GpuKernel,
-
-    X86Fastcall,
-    X86Intr,
-    X86Stdcall,
-    X86ThisCall,
-    X86VectorCall,
-
-    X86_64SysV,
-    X86_64Win64,
-
-    AvrInterrupt,
-    AvrNonBlockingInterrupt,
-
-    RiscvInterrupt { kind: RiscvInterruptKind },
-}
-
-#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)]
 pub enum RiscvInterruptKind {
     Machine,
     Supervisor,
@@ -605,7 +569,7 @@ pub struct FnAbi<'a, Ty> {
     /// This can be used to know whether an argument is variadic or not.
     pub fixed_count: u32,
     /// The calling convention of this function.
-    pub conv: Conv,
+    pub conv: CanonAbi,
     /// Indicates if an unwind may happen across a call to this function.
     pub can_unwind: bool,
 }
@@ -684,7 +648,7 @@ impl<'a, Ty> FnAbi<'a, Ty> {
             "amdgpu" => amdgpu::compute_abi_info(cx, self),
             "arm" => arm::compute_abi_info(cx, self),
             "avr" => avr::compute_abi_info(self),
-            "loongarch64" => loongarch::compute_abi_info(cx, self),
+            "loongarch32" | "loongarch64" => loongarch::compute_abi_info(cx, self),
             "m68k" => m68k::compute_abi_info(self),
             "csky" => csky::compute_abi_info(self),
             "mips" | "mips32r6" => mips::compute_abi_info(cx, self),
@@ -696,7 +660,6 @@ impl<'a, Ty> FnAbi<'a, Ty> {
             "sparc" => sparc::compute_abi_info(cx, self),
             "sparc64" => sparc64::compute_abi_info(cx, self),
             "nvptx64" => {
-                let abi = cx.target_spec().adjust_abi(abi, self.c_variadic);
                 if abi == ExternAbi::PtxKernel || abi == ExternAbi::GpuKernel {
                     nvptx64::compute_ptx_kernel_abi_info(cx, self)
                 } else {
@@ -728,29 +691,11 @@ impl<'a, Ty> FnAbi<'a, Ty> {
         match &*spec.arch {
             "x86" => x86::compute_rust_abi_info(cx, self),
             "riscv32" | "riscv64" => riscv::compute_rust_abi_info(cx, self),
-            "loongarch64" => loongarch::compute_rust_abi_info(cx, self),
+            "loongarch32" | "loongarch64" => loongarch::compute_rust_abi_info(cx, self),
             "aarch64" => aarch64::compute_rust_abi_info(cx, self),
             _ => {}
         };
 
-        // Decides whether we can pass the given SIMD argument via `PassMode::Direct`.
-        // May only return `true` if the target will always pass those arguments the same way,
-        // no matter what the user does with `-Ctarget-feature`! In other words, whatever
-        // target features are required to pass a SIMD value in registers must be listed in
-        // the `abi_required_features` for the current target and ABI.
-        let can_pass_simd_directly = |arg: &ArgAbi<'_, Ty>| match &*spec.arch {
-            // On x86, if we have SSE2 (which we have by default for x86_64), we can always pass up
-            // to 128-bit-sized vectors.
-            "x86" if spec.rustc_abi == Some(RustcAbi::X86Sse2) => arg.layout.size.bits() <= 128,
-            "x86_64" if spec.rustc_abi != Some(RustcAbi::X86Softfloat) => {
-                // FIXME once https://github.com/bytecodealliance/wasmtime/issues/10254 is fixed
-                // accept vectors up to 128bit rather than vectors of exactly 128bit.
-                arg.layout.size.bits() == 128
-            }
-            // So far, we haven't implemented this logic for any other target.
-            _ => false,
-        };
-
         for (arg_idx, arg) in self
             .args
             .iter_mut()
@@ -850,9 +795,10 @@ impl<'a, Ty> FnAbi<'a, Ty> {
                     // target feature sets. Some more information about this
                     // issue can be found in #44367.
                     //
-                    // Note that the intrinsic ABI is exempt here as those are not
-                    // real functions anyway, and the backend expects very specific types.
-                    if spec.simd_types_indirect && !can_pass_simd_directly(arg) {
+                    // We *could* do better in some cases, e.g. on x86_64 targets where SSE2 is
+                    // required. However, it turns out that that makes LLVM worse at optimizing this
+                    // code, so we pass things indirectly even there. See #139029 for more on that.
+                    if spec.simd_types_indirect {
                         arg.make_indirect();
                     }
                 }
@@ -863,70 +809,6 @@ impl<'a, Ty> FnAbi<'a, Ty> {
     }
 }
 
-impl FromStr for Conv {
-    type Err = String;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        match s {
-            "C" => Ok(Conv::C),
-            "Rust" => Ok(Conv::Rust),
-            "RustCold" => Ok(Conv::Rust),
-            "ArmAapcs" => Ok(Conv::ArmAapcs),
-            "CCmseNonSecureCall" => Ok(Conv::CCmseNonSecureCall),
-            "CCmseNonSecureEntry" => Ok(Conv::CCmseNonSecureEntry),
-            "Msp430Intr" => Ok(Conv::Msp430Intr),
-            "X86Fastcall" => Ok(Conv::X86Fastcall),
-            "X86Intr" => Ok(Conv::X86Intr),
-            "X86Stdcall" => Ok(Conv::X86Stdcall),
-            "X86ThisCall" => Ok(Conv::X86ThisCall),
-            "X86VectorCall" => Ok(Conv::X86VectorCall),
-            "X86_64SysV" => Ok(Conv::X86_64SysV),
-            "X86_64Win64" => Ok(Conv::X86_64Win64),
-            "GpuKernel" => Ok(Conv::GpuKernel),
-            "AvrInterrupt" => Ok(Conv::AvrInterrupt),
-            "AvrNonBlockingInterrupt" => Ok(Conv::AvrNonBlockingInterrupt),
-            "RiscvInterrupt(machine)" => {
-                Ok(Conv::RiscvInterrupt { kind: RiscvInterruptKind::Machine })
-            }
-            "RiscvInterrupt(supervisor)" => {
-                Ok(Conv::RiscvInterrupt { kind: RiscvInterruptKind::Supervisor })
-            }
-            _ => Err(format!("'{s}' is not a valid value for entry function call convention.")),
-        }
-    }
-}
-
-fn conv_to_externabi(conv: &Conv) -> ExternAbi {
-    match conv {
-        Conv::C => ExternAbi::C { unwind: false },
-        Conv::Rust => ExternAbi::Rust,
-        Conv::PreserveMost => ExternAbi::RustCold,
-        Conv::ArmAapcs => ExternAbi::Aapcs { unwind: false },
-        Conv::CCmseNonSecureCall => ExternAbi::CCmseNonSecureCall,
-        Conv::CCmseNonSecureEntry => ExternAbi::CCmseNonSecureEntry,
-        Conv::Msp430Intr => ExternAbi::Msp430Interrupt,
-        Conv::GpuKernel => ExternAbi::GpuKernel,
-        Conv::X86Fastcall => ExternAbi::Fastcall { unwind: false },
-        Conv::X86Intr => ExternAbi::X86Interrupt,
-        Conv::X86Stdcall => ExternAbi::Stdcall { unwind: false },
-        Conv::X86ThisCall => ExternAbi::Thiscall { unwind: false },
-        Conv::X86VectorCall => ExternAbi::Vectorcall { unwind: false },
-        Conv::X86_64SysV => ExternAbi::SysV64 { unwind: false },
-        Conv::X86_64Win64 => ExternAbi::Win64 { unwind: false },
-        Conv::AvrInterrupt => ExternAbi::AvrInterrupt,
-        Conv::AvrNonBlockingInterrupt => ExternAbi::AvrNonBlockingInterrupt,
-        Conv::RiscvInterrupt { kind: RiscvInterruptKind::Machine } => ExternAbi::RiscvInterruptM,
-        Conv::RiscvInterrupt { kind: RiscvInterruptKind::Supervisor } => ExternAbi::RiscvInterruptS,
-        Conv::Cold | Conv::PreserveAll => unreachable!(),
-    }
-}
-
-impl Display for Conv {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}", conv_to_externabi(self))
-    }
-}
-
 // Some types are used a lot. Make sure they don't unintentionally get bigger.
 #[cfg(target_pointer_width = "64")]
 mod size_asserts {
diff --git a/compiler/rustc_target/src/json.rs b/compiler/rustc_target/src/json.rs
index 8d6f8f4c6f6..4fcc477921b 100644
--- a/compiler/rustc_target/src/json.rs
+++ b/compiler/rustc_target/src/json.rs
@@ -92,38 +92,6 @@ impl<A: ToJson> ToJson for Option<A> {
     }
 }
 
-impl ToJson for crate::callconv::Conv {
-    fn to_json(&self) -> Json {
-        let buf: String;
-        let s = match self {
-            Self::C => "C",
-            Self::Rust => "Rust",
-            Self::Cold => "Cold",
-            Self::PreserveMost => "PreserveMost",
-            Self::PreserveAll => "PreserveAll",
-            Self::ArmAapcs => "ArmAapcs",
-            Self::CCmseNonSecureCall => "CCmseNonSecureCall",
-            Self::CCmseNonSecureEntry => "CCmseNonSecureEntry",
-            Self::Msp430Intr => "Msp430Intr",
-            Self::X86Fastcall => "X86Fastcall",
-            Self::X86Intr => "X86Intr",
-            Self::X86Stdcall => "X86Stdcall",
-            Self::X86ThisCall => "X86ThisCall",
-            Self::X86VectorCall => "X86VectorCall",
-            Self::X86_64SysV => "X86_64SysV",
-            Self::X86_64Win64 => "X86_64Win64",
-            Self::GpuKernel => "GpuKernel",
-            Self::AvrInterrupt => "AvrInterrupt",
-            Self::AvrNonBlockingInterrupt => "AvrNonBlockingInterrupt",
-            Self::RiscvInterrupt { kind } => {
-                buf = format!("RiscvInterrupt({})", kind.as_str());
-                &buf
-            }
-        };
-        Json::String(s.to_owned())
-    }
-}
-
 impl ToJson for TargetMetadata {
     fn to_json(&self) -> Json {
         json!({
@@ -140,3 +108,9 @@ impl ToJson for rustc_abi::Endian {
         self.as_str().to_json()
     }
 }
+
+impl ToJson for rustc_abi::CanonAbi {
+    fn to_json(&self) -> Json {
+        self.to_string().to_json()
+    }
+}
diff --git a/compiler/rustc_target/src/lib.rs b/compiler/rustc_target/src/lib.rs
index 566bee75c7f..91657fef803 100644
--- a/compiler/rustc_target/src/lib.rs
+++ b/compiler/rustc_target/src/lib.rs
@@ -11,10 +11,8 @@
 #![allow(internal_features)]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
-#![feature(assert_matches)]
 #![feature(debug_closure_helpers)]
 #![feature(iter_intersperse)]
-#![feature(rustc_attrs)]
 #![feature(rustdoc_internals)]
 // tidy-alphabetical-end
 
diff --git a/compiler/rustc_target/src/spec/abi_map.rs b/compiler/rustc_target/src/spec/abi_map.rs
new file mode 100644
index 00000000000..c4978a8e52a
--- /dev/null
+++ b/compiler/rustc_target/src/spec/abi_map.rs
@@ -0,0 +1,185 @@
+use rustc_abi::{ArmCall, CanonAbi, ExternAbi, InterruptKind, X86Call};
+
+use crate::spec::Target;
+
+/// Mapping for ExternAbi to CanonAbi according to a Target
+///
+/// A maybe-transitional structure circa 2025 for hosting future experiments in
+/// encapsulating arch-specific ABI lowering details to make them more testable.
+#[derive(Clone, Debug)]
+pub struct AbiMap {
+    arch: Arch,
+    os: OsKind,
+}
+
+#[derive(Copy, Clone, Debug)]
+pub enum AbiMapping {
+    /// this ABI is exactly mapped for this platform
+    Direct(CanonAbi),
+    /// we don't yet warn on this, but we will
+    Deprecated(CanonAbi),
+    Invalid,
+}
+
+impl AbiMapping {
+    pub fn into_option(self) -> Option<CanonAbi> {
+        match self {
+            Self::Direct(abi) | Self::Deprecated(abi) => Some(abi),
+            Self::Invalid => None,
+        }
+    }
+
+    #[track_caller]
+    pub fn unwrap(self) -> CanonAbi {
+        self.into_option().unwrap()
+    }
+
+    pub fn is_mapped(self) -> bool {
+        self.into_option().is_some()
+    }
+}
+
+impl AbiMap {
+    pub fn from_target(target: &Target) -> Self {
+        // the purpose of this little exercise is to force listing what affects these mappings
+        let arch = match &*target.arch {
+            "aarch64" => Arch::Aarch64,
+            "amdgpu" => Arch::Amdgpu,
+            "arm" if target.llvm_target.starts_with("thumbv8m") => Arch::Arm(ArmVer::ThumbV8M),
+            "arm" => Arch::Arm(ArmVer::Other),
+            "avr" => Arch::Avr,
+            "msp430" => Arch::Msp430,
+            "nvptx64" => Arch::Nvptx,
+            "riscv32" | "riscv64" => Arch::Riscv,
+            "x86" => Arch::X86,
+            "x86_64" => Arch::X86_64,
+            _ => Arch::Other,
+        };
+        let os = if target.is_like_windows { OsKind::Windows } else { OsKind::Other };
+        AbiMap { arch, os }
+    }
+
+    pub fn canonize_abi(&self, extern_abi: ExternAbi, has_c_varargs: bool) -> AbiMapping {
+        let AbiMap { os, arch } = *self;
+
+        let canon_abi = match (extern_abi, arch) {
+            // infallible lowerings
+            (ExternAbi::C { .. }, _) => CanonAbi::C,
+            (ExternAbi::Rust | ExternAbi::RustCall, _) => CanonAbi::Rust,
+            (ExternAbi::Unadjusted, _) => CanonAbi::C,
+
+            (ExternAbi::RustCold, _) if self.os == OsKind::Windows => CanonAbi::Rust,
+            (ExternAbi::RustCold, _) => CanonAbi::RustCold,
+
+            (ExternAbi::System { .. }, Arch::X86) if os == OsKind::Windows && !has_c_varargs => {
+                CanonAbi::X86(X86Call::Stdcall)
+            }
+            (ExternAbi::System { .. }, _) => CanonAbi::C,
+
+            // fallible lowerings
+            (ExternAbi::EfiApi, Arch::Arm(..)) => CanonAbi::Arm(ArmCall::Aapcs),
+            (ExternAbi::EfiApi, Arch::X86_64) => CanonAbi::X86(X86Call::Win64),
+            (ExternAbi::EfiApi, Arch::Aarch64 | Arch::Riscv | Arch::X86) => CanonAbi::C,
+            (ExternAbi::EfiApi, _) => return AbiMapping::Invalid,
+
+            (ExternAbi::Aapcs { .. }, Arch::Arm(..)) => CanonAbi::Arm(ArmCall::Aapcs),
+            (ExternAbi::Aapcs { .. }, _) => return AbiMapping::Invalid,
+
+            (ExternAbi::CCmseNonSecureCall, Arch::Arm(ArmVer::ThumbV8M)) => {
+                CanonAbi::Arm(ArmCall::CCmseNonSecureCall)
+            }
+            (ExternAbi::CCmseNonSecureEntry, Arch::Arm(ArmVer::ThumbV8M)) => {
+                CanonAbi::Arm(ArmCall::CCmseNonSecureEntry)
+            }
+            (ExternAbi::CCmseNonSecureCall | ExternAbi::CCmseNonSecureEntry, ..) => {
+                return AbiMapping::Invalid;
+            }
+
+            (ExternAbi::Cdecl { .. }, Arch::X86) => CanonAbi::C,
+            (ExternAbi::Cdecl { .. }, _) => return AbiMapping::Deprecated(CanonAbi::C),
+
+            (ExternAbi::Fastcall { .. }, Arch::X86) => CanonAbi::X86(X86Call::Fastcall),
+            (ExternAbi::Fastcall { .. }, _) if os == OsKind::Windows => {
+                return AbiMapping::Deprecated(CanonAbi::C);
+            }
+            (ExternAbi::Fastcall { .. }, _) => return AbiMapping::Invalid,
+
+            (ExternAbi::Stdcall { .. }, Arch::X86) => CanonAbi::X86(X86Call::Stdcall),
+            (ExternAbi::Stdcall { .. }, _) if os == OsKind::Windows => {
+                return AbiMapping::Deprecated(CanonAbi::C);
+            }
+            (ExternAbi::Stdcall { .. }, _) => return AbiMapping::Invalid,
+
+            (ExternAbi::Thiscall { .. }, Arch::X86) => CanonAbi::X86(X86Call::Thiscall),
+            (ExternAbi::Thiscall { .. }, _) => return AbiMapping::Invalid,
+
+            (ExternAbi::Vectorcall { .. }, Arch::X86 | Arch::X86_64) => {
+                CanonAbi::X86(X86Call::Vectorcall)
+            }
+            (ExternAbi::Vectorcall { .. }, _) => return AbiMapping::Invalid,
+
+            (ExternAbi::SysV64 { .. }, Arch::X86_64) => CanonAbi::X86(X86Call::SysV64),
+            (ExternAbi::Win64 { .. }, Arch::X86_64) => CanonAbi::X86(X86Call::Win64),
+            (ExternAbi::SysV64 { .. } | ExternAbi::Win64 { .. }, _) => return AbiMapping::Invalid,
+
+            (ExternAbi::PtxKernel, Arch::Nvptx) => CanonAbi::GpuKernel,
+            (ExternAbi::GpuKernel, Arch::Amdgpu | Arch::Nvptx) => CanonAbi::GpuKernel,
+            (ExternAbi::PtxKernel | ExternAbi::GpuKernel, _) => return AbiMapping::Invalid,
+
+            (ExternAbi::AvrInterrupt, Arch::Avr) => CanonAbi::Interrupt(InterruptKind::Avr),
+            (ExternAbi::AvrNonBlockingInterrupt, Arch::Avr) => {
+                CanonAbi::Interrupt(InterruptKind::AvrNonBlocking)
+            }
+            (ExternAbi::Msp430Interrupt, Arch::Msp430) => {
+                CanonAbi::Interrupt(InterruptKind::Msp430)
+            }
+            (ExternAbi::RiscvInterruptM, Arch::Riscv) => {
+                CanonAbi::Interrupt(InterruptKind::RiscvMachine)
+            }
+            (ExternAbi::RiscvInterruptS, Arch::Riscv) => {
+                CanonAbi::Interrupt(InterruptKind::RiscvSupervisor)
+            }
+            (ExternAbi::X86Interrupt, Arch::X86 | Arch::X86_64) => {
+                CanonAbi::Interrupt(InterruptKind::X86)
+            }
+            (
+                ExternAbi::AvrInterrupt
+                | ExternAbi::AvrNonBlockingInterrupt
+                | ExternAbi::Msp430Interrupt
+                | ExternAbi::RiscvInterruptM
+                | ExternAbi::RiscvInterruptS
+                | ExternAbi::X86Interrupt,
+                _,
+            ) => return AbiMapping::Invalid,
+        };
+
+        AbiMapping::Direct(canon_abi)
+    }
+}
+
+#[derive(Debug, PartialEq, Copy, Clone)]
+enum Arch {
+    Aarch64,
+    Amdgpu,
+    Arm(ArmVer),
+    Avr,
+    Msp430,
+    Nvptx,
+    Riscv,
+    X86,
+    X86_64,
+    /// Architectures which don't need other considerations for ABI lowering
+    Other,
+}
+
+#[derive(Debug, PartialEq, Copy, Clone)]
+enum OsKind {
+    Windows,
+    Other,
+}
+
+#[derive(Debug, PartialEq, Copy, Clone)]
+enum ArmVer {
+    ThumbV8M,
+    Other,
+}
diff --git a/compiler/rustc_target/src/spec/base/apple/mod.rs b/compiler/rustc_target/src/spec/base/apple/mod.rs
index 46fcd7d5c51..aa6d1ec7009 100644
--- a/compiler/rustc_target/src/spec/base/apple/mod.rs
+++ b/compiler/rustc_target/src/spec/base/apple/mod.rs
@@ -124,7 +124,13 @@ pub(crate) fn base(
         // to v4, so we do the same.
         // https://github.com/llvm/llvm-project/blob/378778a0d10c2f8d5df8ceff81f95b6002984a4b/clang/lib/Driver/ToolChains/Darwin.cpp#L1203
         default_dwarf_version: 4,
-        frame_pointer: FramePointer::Always,
+        frame_pointer: match arch {
+            // clang ignores `-fomit-frame-pointer` for Armv7, it only accepts `-momit-leaf-frame-pointer`
+            Armv7k | Armv7s => FramePointer::Always,
+            // clang supports omitting frame pointers for the rest, but... don't?
+            Arm64 | Arm64e | Arm64_32 => FramePointer::NonLeaf,
+            I386 | I686 | X86_64 | X86_64h => FramePointer::Always,
+        },
         has_rpath: true,
         dll_suffix: ".dylib".into(),
         archive_format: "darwin".into(),
diff --git a/compiler/rustc_target/src/spec/base/cygwin.rs b/compiler/rustc_target/src/spec/base/cygwin.rs
index 819d1d68a71..d6ae0a905bf 100644
--- a/compiler/rustc_target/src/spec/base/cygwin.rs
+++ b/compiler/rustc_target/src/spec/base/cygwin.rs
@@ -1,7 +1,8 @@
 use std::borrow::Cow;
 
 use crate::spec::{
-    BinaryFormat, Cc, DebuginfoKind, LinkerFlavor, Lld, SplitDebuginfo, TargetOptions, cvs,
+    BinaryFormat, Cc, DebuginfoKind, LinkerFlavor, Lld, SplitDebuginfo, TargetOptions, TlsModel,
+    cvs,
 };
 
 pub(crate) fn opts() -> TargetOptions {
@@ -44,6 +45,8 @@ pub(crate) fn opts() -> TargetOptions {
         eh_frame_header: false,
         debuginfo_kind: DebuginfoKind::Dwarf,
         supported_split_debuginfo: Cow::Borrowed(&[SplitDebuginfo::Off]),
+        tls_model: TlsModel::Emulated,
+        has_thread_local: true,
         ..Default::default()
     }
 }
diff --git a/compiler/rustc_target/src/spec/json.rs b/compiler/rustc_target/src/spec/json.rs
index be71da76b4a..039056a5a25 100644
--- a/compiler/rustc_target/src/spec/json.rs
+++ b/compiler/rustc_target/src/spec/json.rs
@@ -2,10 +2,12 @@ use std::borrow::Cow;
 use std::collections::BTreeMap;
 use std::str::FromStr;
 
+use rustc_abi::{Align, AlignFromBytesError, ExternAbi};
 use serde_json::Value;
 
 use super::{Target, TargetKind, TargetOptions, TargetWarnings};
 use crate::json::{Json, ToJson};
+use crate::spec::AbiMap;
 
 impl Target {
     /// Loads a target descriptor from a JSON object.
@@ -55,6 +57,14 @@ impl Target {
             base.metadata.std = metadata.remove("std").and_then(|host| host.as_bool());
         }
 
+        let alignment_error = |field_name: &str, error: AlignFromBytesError| -> String {
+            let msg = match error {
+                AlignFromBytesError::NotPowerOfTwo(_) => "not a power of 2 number of bytes",
+                AlignFromBytesError::TooLarge(_) => "too large",
+            };
+            format!("`{}` bits is not a valid value for {field_name}: {msg}", error.align() * 8)
+        };
+
         let mut incorrect_type = vec![];
 
         macro_rules! key {
@@ -109,6 +119,15 @@ impl Target {
                     base.$key_name = Some(s.into());
                 }
             } );
+            ($key_name:ident, Option<Align>) => ( {
+                let name = (stringify!($key_name)).replace("_", "-");
+                if let Some(b) = obj.remove(&name).and_then(|b| b.as_u64()) {
+                    match Align::from_bits(b) {
+                        Ok(align) => base.$key_name = Some(align),
+                        Err(e) => return Err(alignment_error(&name, e)),
+                    }
+                }
+            } );
             ($key_name:ident, BinaryFormat) => ( {
                 let name = (stringify!($key_name)).replace("_", "-");
                 obj.remove(&name).and_then(|f| f.as_str().and_then(|s| {
@@ -515,18 +534,6 @@ impl Target {
                     }
                 }
             } );
-            ($key_name:ident, Conv) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                obj.remove(&name).and_then(|o| o.as_str().and_then(|s| {
-                    match super::Conv::from_str(s) {
-                        Ok(c) => {
-                            base.$key_name = c;
-                            Some(Ok(()))
-                        }
-                        Err(e) => Some(Err(e))
-                    }
-                })).unwrap_or(Ok(()))
-            } );
         }
 
         if let Some(j) = obj.remove("target-endian") {
@@ -627,7 +634,7 @@ impl Target {
         key!(crt_static_default, bool);
         key!(crt_static_respected, bool);
         key!(stack_probes, StackProbeType)?;
-        key!(min_global_align, Option<u64>);
+        key!(min_global_align, Option<Align>);
         key!(default_codegen_units, Option<u64>);
         key!(default_codegen_backend, Option<StaticCow<str>>);
         key!(trap_unreachable, bool);
@@ -660,9 +667,23 @@ impl Target {
         key!(supports_stack_protector, bool);
         key!(small_data_threshold_support, SmallDataThresholdSupport)?;
         key!(entry_name);
-        key!(entry_abi, Conv)?;
         key!(supports_xray, bool);
 
+        // we're going to run `update_from_cli`, but that won't change the target's AbiMap
+        // FIXME: better factor the Target definition so we enforce this on a type level
+        let abi_map = AbiMap::from_target(&base);
+
+        if let Some(abi_str) = obj.remove("entry-abi") {
+            if let Json::String(abi_str) = abi_str {
+                match abi_str.parse::<ExternAbi>() {
+                    Ok(abi) => base.options.entry_abi = abi_map.canonize_abi(abi, false).unwrap(),
+                    Err(_) => return Err(format!("{abi_str} is not a valid ExternAbi")),
+                }
+            } else {
+                incorrect_type.push("entry-abi".to_owned())
+            }
+        }
+
         base.update_from_cli();
         base.check_consistency(TargetKind::Json)?;
 
diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
index 303be54a6d7..6ceb5e80f21 100644
--- a/compiler/rustc_target/src/spec/mod.rs
+++ b/compiler/rustc_target/src/spec/mod.rs
@@ -43,7 +43,7 @@ use std::str::FromStr;
 use std::{fmt, io};
 
 use rustc_abi::{
-    Align, Endian, ExternAbi, Integer, Size, TargetDataLayout, TargetDataLayoutErrors,
+    Align, CanonAbi, Endian, ExternAbi, Integer, Size, TargetDataLayout, TargetDataLayoutErrors,
 };
 use rustc_data_structures::fx::{FxHashSet, FxIndexSet};
 use rustc_fs_util::try_canonicalize;
@@ -53,15 +53,16 @@ use rustc_span::{Symbol, kw, sym};
 use serde_json::Value;
 use tracing::debug;
 
-use crate::callconv::Conv;
 use crate::json::{Json, ToJson};
 use crate::spec::crt_objects::CrtObjects;
 
 pub mod crt_objects;
 
+mod abi_map;
 mod base;
 mod json;
 
+pub use abi_map::{AbiMap, AbiMapping};
 pub use base::apple;
 pub use base::avr::ef_avr_arch;
 
@@ -1696,6 +1697,12 @@ impl ToJson for BinaryFormat {
     }
 }
 
+impl ToJson for Align {
+    fn to_json(&self) -> Json {
+        self.bits().to_json()
+    }
+}
+
 macro_rules! supported_targets {
     ( $(($tuple:literal, $module:ident),)+ ) => {
         mod targets {
@@ -1980,6 +1987,8 @@ supported_targets! {
 
     ("sparc-unknown-none-elf", sparc_unknown_none_elf),
 
+    ("loongarch32-unknown-none", loongarch32_unknown_none),
+    ("loongarch32-unknown-none-softfloat", loongarch32_unknown_none_softfloat),
     ("loongarch64-unknown-none", loongarch64_unknown_none),
     ("loongarch64-unknown-none-softfloat", loongarch64_unknown_none_softfloat),
 
@@ -2512,7 +2521,7 @@ pub struct TargetOptions {
     pub stack_probes: StackProbeType,
 
     /// The minimum alignment for global symbols.
-    pub min_global_align: Option<u64>,
+    pub min_global_align: Option<Align>,
 
     /// Default number of codegen units to use in debug mode
     pub default_codegen_units: Option<u64>,
@@ -2655,9 +2664,9 @@ pub struct TargetOptions {
     /// Default value is "main"
     pub entry_name: StaticCow<str>,
 
-    /// The ABI of entry function.
-    /// Default value is `Conv::C`, i.e. C call convention
-    pub entry_abi: Conv,
+    /// The ABI of the entry function.
+    /// Default value is `CanonAbi::C`
+    pub entry_abi: CanonAbi,
 
     /// Whether the target supports XRay instrumentation.
     pub supports_xray: bool,
@@ -2888,7 +2897,7 @@ impl Default for TargetOptions {
             generate_arange_section: true,
             supports_stack_protector: true,
             entry_name: "main".into(),
-            entry_abi: Conv::C,
+            entry_abi: CanonAbi::C,
             supports_xray: false,
             small_data_threshold_support: SmallDataThresholdSupport::DefaultForArch,
         }
@@ -2914,114 +2923,9 @@ impl DerefMut for Target {
 }
 
 impl Target {
-    /// Given a function ABI, turn it into the correct ABI for this target.
-    pub fn adjust_abi(&self, abi: ExternAbi, c_variadic: bool) -> ExternAbi {
-        use ExternAbi::*;
-        match abi {
-            // On Windows, `extern "system"` behaves like msvc's `__stdcall`.
-            // `__stdcall` only applies on x86 and on non-variadic functions:
-            // https://learn.microsoft.com/en-us/cpp/cpp/stdcall?view=msvc-170
-            System { unwind } => {
-                if self.is_like_windows && self.arch == "x86" && !c_variadic {
-                    Stdcall { unwind }
-                } else {
-                    C { unwind }
-                }
-            }
-
-            EfiApi => {
-                if self.arch == "arm" {
-                    Aapcs { unwind: false }
-                } else if self.arch == "x86_64" {
-                    Win64 { unwind: false }
-                } else {
-                    C { unwind: false }
-                }
-            }
-
-            // See commentary in `is_abi_supported`.
-            Stdcall { unwind } | Thiscall { unwind } | Fastcall { unwind } => {
-                if self.arch == "x86" { abi } else { C { unwind } }
-            }
-            Vectorcall { unwind } => {
-                if ["x86", "x86_64"].contains(&&*self.arch) {
-                    abi
-                } else {
-                    C { unwind }
-                }
-            }
-
-            // The Windows x64 calling convention we use for `extern "Rust"`
-            // <https://learn.microsoft.com/en-us/cpp/build/x64-software-conventions#register-volatility-and-preservation>
-            // expects the callee to save `xmm6` through `xmm15`, but `PreserveMost`
-            // (that we use by default for `extern "rust-cold"`) doesn't save any of those.
-            // So to avoid bloating callers, just use the Rust convention here.
-            RustCold if self.is_like_windows && self.arch == "x86_64" => Rust,
-
-            abi => abi,
-        }
-    }
-
     pub fn is_abi_supported(&self, abi: ExternAbi) -> bool {
-        use ExternAbi::*;
-        match abi {
-            Rust | C { .. } | System { .. } | RustCall | Unadjusted | Cdecl { .. } | RustCold => {
-                true
-            }
-            EfiApi => {
-                ["arm", "aarch64", "riscv32", "riscv64", "x86", "x86_64"].contains(&&self.arch[..])
-            }
-            X86Interrupt => ["x86", "x86_64"].contains(&&self.arch[..]),
-            Aapcs { .. } => "arm" == self.arch,
-            CCmseNonSecureCall | CCmseNonSecureEntry => {
-                ["thumbv8m.main-none-eabi", "thumbv8m.main-none-eabihf", "thumbv8m.base-none-eabi"]
-                    .contains(&&self.llvm_target[..])
-            }
-            Win64 { .. } | SysV64 { .. } => self.arch == "x86_64",
-            PtxKernel => self.arch == "nvptx64",
-            GpuKernel => ["amdgpu", "nvptx64"].contains(&&self.arch[..]),
-            Msp430Interrupt => self.arch == "msp430",
-            RiscvInterruptM | RiscvInterruptS => ["riscv32", "riscv64"].contains(&&self.arch[..]),
-            AvrInterrupt | AvrNonBlockingInterrupt => self.arch == "avr",
-            Thiscall { .. } => self.arch == "x86",
-            // On windows these fall-back to platform native calling convention (C) when the
-            // architecture is not supported.
-            //
-            // This is I believe a historical accident that has occurred as part of Microsoft
-            // striving to allow most of the code to "just" compile when support for 64-bit x86
-            // was added and then later again, when support for ARM architectures was added.
-            //
-            // This is well documented across MSDN. Support for this in Rust has been added in
-            // #54576. This makes much more sense in context of Microsoft's C++ than it does in
-            // Rust, but there isn't much leeway remaining here to change it back at the time this
-            // comment has been written.
-            //
-            // Following are the relevant excerpts from the MSDN documentation.
-            //
-            // > The __vectorcall calling convention is only supported in native code on x86 and
-            // x64 processors that include Streaming SIMD Extensions 2 (SSE2) and above.
-            // > ...
-            // > On ARM machines, __vectorcall is accepted and ignored by the compiler.
-            //
-            // -- https://docs.microsoft.com/en-us/cpp/cpp/vectorcall?view=msvc-160
-            //
-            // > On ARM and x64 processors, __stdcall is accepted and ignored by the compiler;
-            //
-            // -- https://docs.microsoft.com/en-us/cpp/cpp/stdcall?view=msvc-160
-            //
-            // > In most cases, keywords or compiler switches that specify an unsupported
-            // > convention on a particular platform are ignored, and the platform default
-            // > convention is used.
-            //
-            // -- https://docs.microsoft.com/en-us/cpp/cpp/argument-passing-and-naming-conventions
-            Stdcall { .. } | Fastcall { .. } | Vectorcall { .. } if self.is_like_windows => true,
-            // Outside of Windows we want to only support these calling conventions for the
-            // architectures for which these calling conventions are actually well defined.
-            Stdcall { .. } | Fastcall { .. } if self.arch == "x86" => true,
-            Vectorcall { .. } if ["x86", "x86_64"].contains(&&self.arch[..]) => true,
-            // Reject these calling conventions everywhere else.
-            Stdcall { .. } | Fastcall { .. } | Vectorcall { .. } => false,
-        }
+        let abi_map = AbiMap::from_target(self);
+        abi_map.canonize_abi(abi, false).is_mapped()
     }
 
     /// Minimum integer size in bits that this target can perform atomic
@@ -3606,6 +3510,7 @@ impl Target {
             "msp430" => (Architecture::Msp430, None),
             "hexagon" => (Architecture::Hexagon, None),
             "bpf" => (Architecture::Bpf, None),
+            "loongarch32" => (Architecture::LoongArch32, None),
             "loongarch64" => (Architecture::LoongArch64, None),
             "csky" => (Architecture::Csky, None),
             "arm64ec" => (Architecture::Aarch64, Some(object::SubArchitecture::Arm64EC)),
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_darwin.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_darwin.rs
index d3e0a32c8b8..4dd39877715 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_darwin.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_darwin.rs
@@ -1,5 +1,5 @@
 use crate::spec::base::apple::{Arch, TargetAbi, base};
-use crate::spec::{FramePointer, SanitizerSet, Target, TargetMetadata, TargetOptions};
+use crate::spec::{SanitizerSet, Target, TargetMetadata, TargetOptions};
 
 pub(crate) fn target() -> Target {
     let (opts, llvm_target, arch) = base("macos", Arch::Arm64, TargetAbi::Normal);
@@ -17,11 +17,11 @@ pub(crate) fn target() -> Target {
         arch,
         options: TargetOptions {
             mcount: "\u{1}mcount".into(),
-            frame_pointer: FramePointer::NonLeaf,
             cpu: "apple-m1".into(),
             max_atomic_width: Some(128),
             // FIXME: The leak sanitizer currently fails the tests, see #88132.
             supported_sanitizers: SanitizerSet::ADDRESS | SanitizerSet::CFI | SanitizerSet::THREAD,
+            supports_xray: true,
             ..opts
         },
     }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios.rs
index 183a6c6f2d7..769a7b6c391 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios.rs
@@ -1,5 +1,5 @@
 use crate::spec::base::apple::{Arch, TargetAbi, base};
-use crate::spec::{FramePointer, SanitizerSet, Target, TargetMetadata, TargetOptions};
+use crate::spec::{SanitizerSet, Target, TargetMetadata, TargetOptions};
 
 pub(crate) fn target() -> Target {
     let (opts, llvm_target, arch) = base("ios", Arch::Arm64, TargetAbi::Normal);
@@ -18,7 +18,6 @@ pub(crate) fn target() -> Target {
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a7".into(),
             max_atomic_width: Some(128),
-            frame_pointer: FramePointer::NonLeaf,
             supported_sanitizers: SanitizerSet::ADDRESS | SanitizerSet::THREAD,
             ..opts
         },
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_macabi.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_macabi.rs
index ce9ae03e699..4bb2f73e4f9 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_macabi.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_macabi.rs
@@ -1,5 +1,5 @@
 use crate::spec::base::apple::{Arch, TargetAbi, base};
-use crate::spec::{FramePointer, SanitizerSet, Target, TargetMetadata, TargetOptions};
+use crate::spec::{SanitizerSet, Target, TargetMetadata, TargetOptions};
 
 pub(crate) fn target() -> Target {
     let (opts, llvm_target, arch) = base("ios", Arch::Arm64, TargetAbi::MacCatalyst);
@@ -18,7 +18,6 @@ pub(crate) fn target() -> Target {
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a12".into(),
             max_atomic_width: Some(128),
-            frame_pointer: FramePointer::NonLeaf,
             supported_sanitizers: SanitizerSet::ADDRESS | SanitizerSet::LEAK | SanitizerSet::THREAD,
             ..opts
         },
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_sim.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_sim.rs
index 4405e3fec02..7d04034e759 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_sim.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_sim.rs
@@ -1,5 +1,5 @@
 use crate::spec::base::apple::{Arch, TargetAbi, base};
-use crate::spec::{FramePointer, SanitizerSet, Target, TargetMetadata, TargetOptions};
+use crate::spec::{SanitizerSet, Target, TargetMetadata, TargetOptions};
 
 pub(crate) fn target() -> Target {
     let (opts, llvm_target, arch) = base("ios", Arch::Arm64, TargetAbi::Simulator);
@@ -18,7 +18,6 @@ pub(crate) fn target() -> Target {
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a7".into(),
             max_atomic_width: Some(128),
-            frame_pointer: FramePointer::NonLeaf,
             supported_sanitizers: SanitizerSet::ADDRESS | SanitizerSet::THREAD,
             ..opts
         },
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos.rs
index 037685db1b3..ec92a40e255 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos.rs
@@ -1,5 +1,5 @@
 use crate::spec::base::apple::{Arch, TargetAbi, base};
-use crate::spec::{FramePointer, Target, TargetMetadata, TargetOptions};
+use crate::spec::{Target, TargetMetadata, TargetOptions};
 
 pub(crate) fn target() -> Target {
     let (opts, llvm_target, arch) = base("tvos", Arch::Arm64, TargetAbi::Normal);
@@ -18,7 +18,6 @@ pub(crate) fn target() -> Target {
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a7".into(),
             max_atomic_width: Some(128),
-            frame_pointer: FramePointer::NonLeaf,
             ..opts
         },
     }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos_sim.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos_sim.rs
index a386220e6fc..74fbe5a89ca 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos_sim.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos_sim.rs
@@ -1,5 +1,5 @@
 use crate::spec::base::apple::{Arch, TargetAbi, base};
-use crate::spec::{FramePointer, Target, TargetMetadata, TargetOptions};
+use crate::spec::{Target, TargetMetadata, TargetOptions};
 
 pub(crate) fn target() -> Target {
     let (opts, llvm_target, arch) = base("tvos", Arch::Arm64, TargetAbi::Simulator);
@@ -18,7 +18,6 @@ pub(crate) fn target() -> Target {
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a7".into(),
             max_atomic_width: Some(128),
-            frame_pointer: FramePointer::NonLeaf,
             ..opts
         },
     }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos.rs
index 2c1dfdd55ed..dc595fbe7b6 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos.rs
@@ -1,5 +1,5 @@
 use crate::spec::base::apple::{Arch, TargetAbi, base};
-use crate::spec::{FramePointer, SanitizerSet, Target, TargetMetadata, TargetOptions};
+use crate::spec::{SanitizerSet, Target, TargetMetadata, TargetOptions};
 
 pub(crate) fn target() -> Target {
     let (opts, llvm_target, arch) = base("visionos", Arch::Arm64, TargetAbi::Normal);
@@ -18,7 +18,6 @@ pub(crate) fn target() -> Target {
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a16".into(),
             max_atomic_width: Some(128),
-            frame_pointer: FramePointer::NonLeaf,
             supported_sanitizers: SanitizerSet::ADDRESS | SanitizerSet::THREAD,
             ..opts
         },
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos_sim.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos_sim.rs
index c0b8b409797..06ff1bfb2f0 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos_sim.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos_sim.rs
@@ -1,5 +1,5 @@
 use crate::spec::base::apple::{Arch, TargetAbi, base};
-use crate::spec::{FramePointer, SanitizerSet, Target, TargetMetadata, TargetOptions};
+use crate::spec::{SanitizerSet, Target, TargetMetadata, TargetOptions};
 
 pub(crate) fn target() -> Target {
     let (opts, llvm_target, arch) = base("visionos", Arch::Arm64, TargetAbi::Simulator);
@@ -18,7 +18,6 @@ pub(crate) fn target() -> Target {
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a16".into(),
             max_atomic_width: Some(128),
-            frame_pointer: FramePointer::NonLeaf,
             supported_sanitizers: SanitizerSet::ADDRESS | SanitizerSet::THREAD,
             ..opts
         },
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos_sim.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos_sim.rs
index 62968f5b555..bad9f6c1485 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos_sim.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos_sim.rs
@@ -1,5 +1,5 @@
 use crate::spec::base::apple::{Arch, TargetAbi, base};
-use crate::spec::{FramePointer, Target, TargetMetadata, TargetOptions};
+use crate::spec::{Target, TargetMetadata, TargetOptions};
 
 pub(crate) fn target() -> Target {
     let (opts, llvm_target, arch) = base("watchos", Arch::Arm64, TargetAbi::Simulator);
@@ -18,7 +18,6 @@ pub(crate) fn target() -> Target {
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a7".into(),
             max_atomic_width: Some(128),
-            frame_pointer: FramePointer::NonLeaf,
             ..opts
         },
     }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_be_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/targets/aarch64_be_unknown_linux_gnu.rs
index 87c07cd3109..4b75a6e5dbf 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_be_unknown_linux_gnu.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_be_unknown_linux_gnu.rs
@@ -1,6 +1,6 @@
 use rustc_abi::Endian;
 
-use crate::spec::{StackProbeType, Target, TargetMetadata, TargetOptions, base};
+use crate::spec::{FramePointer, StackProbeType, Target, TargetMetadata, TargetOptions, base};
 
 pub(crate) fn target() -> Target {
     Target {
@@ -16,6 +16,10 @@ pub(crate) fn target() -> Target {
         arch: "aarch64".into(),
         options: TargetOptions {
             features: "+v8a,+outline-atomics".into(),
+            // the AAPCS64 expects use of non-leaf frame pointers per
+            // https://github.com/ARM-software/abi-aa/blob/4492d1570eb70c8fd146623e0db65b2d241f12e7/aapcs64/aapcs64.rst#the-frame-pointer
+            // and we tend to encounter interesting bugs in AArch64 unwinding code if we do not
+            frame_pointer: FramePointer::NonLeaf,
             max_atomic_width: Some(128),
             stack_probes: StackProbeType::Inline,
             mcount: "\u{1}_mcount".into(),
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_be_unknown_linux_gnu_ilp32.rs b/compiler/rustc_target/src/spec/targets/aarch64_be_unknown_linux_gnu_ilp32.rs
index e785069c78a..2a16d1de3b5 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_be_unknown_linux_gnu_ilp32.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_be_unknown_linux_gnu_ilp32.rs
@@ -1,6 +1,6 @@
 use rustc_abi::Endian;
 
-use crate::spec::{StackProbeType, Target, TargetMetadata, TargetOptions, base};
+use crate::spec::{FramePointer, StackProbeType, Target, TargetMetadata, TargetOptions, base};
 
 pub(crate) fn target() -> Target {
     let mut base = base::linux_gnu::opts();
@@ -20,6 +20,10 @@ pub(crate) fn target() -> Target {
         options: TargetOptions {
             abi: "ilp32".into(),
             features: "+v8a,+outline-atomics".into(),
+            // the AAPCS64 expects use of non-leaf frame pointers per
+            // https://github.com/ARM-software/abi-aa/blob/4492d1570eb70c8fd146623e0db65b2d241f12e7/aapcs64/aapcs64.rst#the-frame-pointer
+            // and we tend to encounter interesting bugs in AArch64 unwinding code if we do not
+            frame_pointer: FramePointer::NonLeaf,
             stack_probes: StackProbeType::Inline,
             mcount: "\u{1}_mcount".into(),
             endian: Endian::Big,
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_linux_android.rs b/compiler/rustc_target/src/spec/targets/aarch64_linux_android.rs
index 41c25393e12..d8651f305fe 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_linux_android.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_linux_android.rs
@@ -1,4 +1,6 @@
-use crate::spec::{SanitizerSet, StackProbeType, Target, TargetMetadata, TargetOptions, base};
+use crate::spec::{
+    FramePointer, SanitizerSet, StackProbeType, Target, TargetMetadata, TargetOptions, base,
+};
 
 // See https://developer.android.com/ndk/guides/abis.html#arm64-v8a
 // for target ABI requirements.
@@ -20,6 +22,10 @@ pub(crate) fn target() -> Target {
             // As documented in https://developer.android.com/ndk/guides/cpu-features.html
             // the neon (ASIMD) and FP must exist on all android aarch64 targets.
             features: "+v8a,+neon,+fp-armv8".into(),
+            // the AAPCS64 expects use of non-leaf frame pointers per
+            // https://github.com/ARM-software/abi-aa/blob/4492d1570eb70c8fd146623e0db65b2d241f12e7/aapcs64/aapcs64.rst#the-frame-pointer
+            // and we tend to encounter interesting bugs in AArch64 unwinding code if we do not
+            frame_pointer: FramePointer::NonLeaf,
             stack_probes: StackProbeType::Inline,
             supported_sanitizers: SanitizerSet::CFI
                 | SanitizerSet::HWADDRESS
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_pc_windows_gnullvm.rs b/compiler/rustc_target/src/spec/targets/aarch64_pc_windows_gnullvm.rs
index a8b133d19bb..eee668cc67e 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_pc_windows_gnullvm.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_pc_windows_gnullvm.rs
@@ -1,4 +1,4 @@
-use crate::spec::{Target, TargetMetadata, base};
+use crate::spec::{FramePointer, Target, TargetMetadata, base};
 
 pub(crate) fn target() -> Target {
     let mut base = base::windows_gnullvm::opts();
@@ -6,6 +6,12 @@ pub(crate) fn target() -> Target {
     base.features = "+v8a,+neon,+fp-armv8".into();
     base.linker = Some("aarch64-w64-mingw32-clang".into());
 
+    // Microsoft recommends enabling frame pointers on Arm64 Windows.
+    // From https://learn.microsoft.com/en-us/cpp/build/arm64-windows-abi-conventions?view=msvc-170#integer-registers
+    // "The frame pointer (x29) is required for compatibility with fast stack walking used by ETW
+    // and other services. It must point to the previous {x29, x30} pair on the stack."
+    base.frame_pointer = FramePointer::NonLeaf;
+
     Target {
         llvm_target: "aarch64-pc-windows-gnu".into(),
         metadata: TargetMetadata {
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_gnu.rs
index c6be2c20ea2..4220d74dfc8 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_gnu.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_gnu.rs
@@ -1,4 +1,6 @@
-use crate::spec::{SanitizerSet, StackProbeType, Target, TargetMetadata, TargetOptions, base};
+use crate::spec::{
+    FramePointer, SanitizerSet, StackProbeType, Target, TargetMetadata, TargetOptions, base,
+};
 
 pub(crate) fn target() -> Target {
     Target {
@@ -14,6 +16,10 @@ pub(crate) fn target() -> Target {
         arch: "aarch64".into(),
         options: TargetOptions {
             features: "+v8a,+outline-atomics".into(),
+            // the AAPCS64 expects use of non-leaf frame pointers per
+            // https://github.com/ARM-software/abi-aa/blob/4492d1570eb70c8fd146623e0db65b2d241f12e7/aapcs64/aapcs64.rst#the-frame-pointer
+            // and we tend to encounter interesting bugs in AArch64 unwinding code if we do not
+            frame_pointer: FramePointer::NonLeaf,
             mcount: "\u{1}_mcount".into(),
             max_atomic_width: Some(128),
             stack_probes: StackProbeType::Inline,
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_gnu_ilp32.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_gnu_ilp32.rs
index 166bb1ed215..a22c1289677 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_gnu_ilp32.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_gnu_ilp32.rs
@@ -1,4 +1,4 @@
-use crate::spec::{StackProbeType, Target, TargetMetadata, TargetOptions, base};
+use crate::spec::{FramePointer, StackProbeType, Target, TargetMetadata, TargetOptions, base};
 
 pub(crate) fn target() -> Target {
     Target {
@@ -15,6 +15,10 @@ pub(crate) fn target() -> Target {
         options: TargetOptions {
             abi: "ilp32".into(),
             features: "+v8a,+outline-atomics".into(),
+            // the AAPCS64 expects use of non-leaf frame pointers per
+            // https://github.com/ARM-software/abi-aa/blob/4492d1570eb70c8fd146623e0db65b2d241f12e7/aapcs64/aapcs64.rst#the-frame-pointer
+            // and we tend to encounter interesting bugs in AArch64 unwinding code if we do not
+            frame_pointer: FramePointer::NonLeaf,
             max_atomic_width: Some(128),
             stack_probes: StackProbeType::Inline,
             mcount: "\u{1}_mcount".into(),
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_musl.rs
index 58ba06e124c..58daaa03675 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_musl.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_musl.rs
@@ -1,4 +1,6 @@
-use crate::spec::{SanitizerSet, StackProbeType, Target, TargetMetadata, TargetOptions, base};
+use crate::spec::{
+    FramePointer, SanitizerSet, StackProbeType, Target, TargetMetadata, TargetOptions, base,
+};
 
 pub(crate) fn target() -> Target {
     let mut base = base::linux_musl::opts();
@@ -26,6 +28,12 @@ pub(crate) fn target() -> Target {
         pointer_width: 64,
         data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(),
         arch: "aarch64".into(),
-        options: TargetOptions { mcount: "\u{1}_mcount".into(), ..base },
+        options: TargetOptions {
+            // the AAPCS64 expects use of non-leaf frame pointers per
+            // https://github.com/ARM-software/abi-aa/blob/4492d1570eb70c8fd146623e0db65b2d241f12e7/aapcs64/aapcs64.rst#the-frame-pointer
+            // and we tend to encounter interesting bugs in AArch64 unwinding code if we do not
+            frame_pointer: FramePointer::NonLeaf,
+             mcount: "\u{1}_mcount".into(), ..base
+         },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_ohos.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_ohos.rs
index f2994b1232e..51cdebf22db 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_ohos.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_ohos.rs
@@ -1,4 +1,6 @@
-use crate::spec::{SanitizerSet, StackProbeType, Target, TargetMetadata, TargetOptions, base};
+use crate::spec::{
+    FramePointer, SanitizerSet, StackProbeType, Target, TargetMetadata, TargetOptions, base,
+};
 
 pub(crate) fn target() -> Target {
     let mut base = base::linux_ohos::opts();
@@ -16,6 +18,10 @@ pub(crate) fn target() -> Target {
         data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32".into(),
         arch: "aarch64".into(),
         options: TargetOptions {
+            // the AAPCS64 expects use of non-leaf frame pointers per
+            // https://github.com/ARM-software/abi-aa/blob/4492d1570eb70c8fd146623e0db65b2d241f12e7/aapcs64/aapcs64.rst#the-frame-pointer
+            // and we tend to encounter interesting bugs in AArch64 unwinding code if we do not
+            frame_pointer: FramePointer::NonLeaf,
             mcount: "\u{1}_mcount".into(),
             stack_probes: StackProbeType::Inline,
             supported_sanitizers: SanitizerSet::ADDRESS
diff --git a/compiler/rustc_target/src/spec/targets/arm64e_apple_darwin.rs b/compiler/rustc_target/src/spec/targets/arm64e_apple_darwin.rs
index 79b95dbde52..326f2b16d59 100644
--- a/compiler/rustc_target/src/spec/targets/arm64e_apple_darwin.rs
+++ b/compiler/rustc_target/src/spec/targets/arm64e_apple_darwin.rs
@@ -1,5 +1,5 @@
 use crate::spec::base::apple::{Arch, TargetAbi, base};
-use crate::spec::{FramePointer, SanitizerSet, Target, TargetMetadata, TargetOptions};
+use crate::spec::{SanitizerSet, Target, TargetMetadata, TargetOptions};
 
 pub(crate) fn target() -> Target {
     let (opts, llvm_target, arch) = base("macos", Arch::Arm64e, TargetAbi::Normal);
@@ -17,7 +17,6 @@ pub(crate) fn target() -> Target {
         arch,
         options: TargetOptions {
             mcount: "\u{1}mcount".into(),
-            frame_pointer: FramePointer::NonLeaf,
             cpu: "apple-m1".into(),
             max_atomic_width: Some(128),
             // FIXME: The leak sanitizer currently fails the tests, see #88132.
diff --git a/compiler/rustc_target/src/spec/targets/arm64e_apple_ios.rs b/compiler/rustc_target/src/spec/targets/arm64e_apple_ios.rs
index 848dbeec199..01c6f0b888d 100644
--- a/compiler/rustc_target/src/spec/targets/arm64e_apple_ios.rs
+++ b/compiler/rustc_target/src/spec/targets/arm64e_apple_ios.rs
@@ -1,5 +1,5 @@
 use crate::spec::base::apple::{Arch, TargetAbi, base};
-use crate::spec::{FramePointer, SanitizerSet, Target, TargetMetadata, TargetOptions};
+use crate::spec::{SanitizerSet, Target, TargetMetadata, TargetOptions};
 
 pub(crate) fn target() -> Target {
     let (opts, llvm_target, arch) = base("ios", Arch::Arm64e, TargetAbi::Normal);
@@ -18,7 +18,6 @@ pub(crate) fn target() -> Target {
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a12,+v8.3a,+pauth".into(),
             max_atomic_width: Some(128),
-            frame_pointer: FramePointer::NonLeaf,
             supported_sanitizers: SanitizerSet::ADDRESS | SanitizerSet::THREAD,
             ..opts
         },
diff --git a/compiler/rustc_target/src/spec/targets/arm64e_apple_tvos.rs b/compiler/rustc_target/src/spec/targets/arm64e_apple_tvos.rs
index 3dbe169e826..cad3650bda1 100644
--- a/compiler/rustc_target/src/spec/targets/arm64e_apple_tvos.rs
+++ b/compiler/rustc_target/src/spec/targets/arm64e_apple_tvos.rs
@@ -1,5 +1,5 @@
 use crate::spec::base::apple::{Arch, TargetAbi, base};
-use crate::spec::{FramePointer, Target, TargetMetadata, TargetOptions};
+use crate::spec::{Target, TargetMetadata, TargetOptions};
 
 pub(crate) fn target() -> Target {
     let (opts, llvm_target, arch) = base("tvos", Arch::Arm64e, TargetAbi::Normal);
@@ -18,7 +18,6 @@ pub(crate) fn target() -> Target {
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a12,+v8.3a,+pauth".into(),
             max_atomic_width: Some(128),
-            frame_pointer: FramePointer::NonLeaf,
             ..opts
         },
     }
diff --git a/compiler/rustc_target/src/spec/targets/i686_apple_darwin.rs b/compiler/rustc_target/src/spec/targets/i686_apple_darwin.rs
index 161db9a08bb..d1339c57b00 100644
--- a/compiler/rustc_target/src/spec/targets/i686_apple_darwin.rs
+++ b/compiler/rustc_target/src/spec/targets/i686_apple_darwin.rs
@@ -1,5 +1,5 @@
 use crate::spec::base::apple::{Arch, TargetAbi, base};
-use crate::spec::{FramePointer, Target, TargetMetadata, TargetOptions};
+use crate::spec::{Target, TargetMetadata, TargetOptions};
 
 pub(crate) fn target() -> Target {
     let (opts, llvm_target, arch) = base("macos", Arch::I686, TargetAbi::Normal);
@@ -16,11 +16,6 @@ pub(crate) fn target() -> Target {
             i128:128-f64:32:64-f80:128-n8:16:32-S128"
             .into(),
         arch,
-        options: TargetOptions {
-            mcount: "\u{1}mcount".into(),
-            max_atomic_width: Some(64),
-            frame_pointer: FramePointer::Always,
-            ..opts
-        },
+        options: TargetOptions { mcount: "\u{1}mcount".into(), max_atomic_width: Some(64), ..opts },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/loongarch32_unknown_none.rs b/compiler/rustc_target/src/spec/targets/loongarch32_unknown_none.rs
new file mode 100644
index 00000000000..fb4963b88b0
--- /dev/null
+++ b/compiler/rustc_target/src/spec/targets/loongarch32_unknown_none.rs
@@ -0,0 +1,29 @@
+use crate::spec::{
+    Cc, LinkerFlavor, Lld, PanicStrategy, RelocModel, Target, TargetMetadata, TargetOptions,
+};
+
+pub(crate) fn target() -> Target {
+    Target {
+        llvm_target: "loongarch32-unknown-none".into(),
+        metadata: TargetMetadata {
+            description: Some("Freestanding/bare-metal LoongArch32".into()),
+            tier: Some(3),
+            host_tools: Some(false),
+            std: Some(false),
+        },
+        pointer_width: 32,
+        data_layout: "e-m:e-p:32:32-i64:64-n32-S128".into(),
+        arch: "loongarch32".into(),
+        options: TargetOptions {
+            cpu: "generic".into(),
+            features: "+f,+d".into(),
+            linker_flavor: LinkerFlavor::Gnu(Cc::No, Lld::Yes),
+            linker: Some("rust-lld".into()),
+            llvm_abiname: "ilp32d".into(),
+            max_atomic_width: Some(32),
+            relocation_model: RelocModel::Static,
+            panic_strategy: PanicStrategy::Abort,
+            ..Default::default()
+        },
+    }
+}
diff --git a/compiler/rustc_target/src/spec/targets/loongarch32_unknown_none_softfloat.rs b/compiler/rustc_target/src/spec/targets/loongarch32_unknown_none_softfloat.rs
new file mode 100644
index 00000000000..0e65f83a71c
--- /dev/null
+++ b/compiler/rustc_target/src/spec/targets/loongarch32_unknown_none_softfloat.rs
@@ -0,0 +1,30 @@
+use crate::spec::{
+    Cc, LinkerFlavor, Lld, PanicStrategy, RelocModel, Target, TargetMetadata, TargetOptions,
+};
+
+pub(crate) fn target() -> Target {
+    Target {
+        llvm_target: "loongarch32-unknown-none".into(),
+        metadata: TargetMetadata {
+            description: Some("Freestanding/bare-metal LoongArch32 softfloat".into()),
+            tier: Some(3),
+            host_tools: Some(false),
+            std: Some(false),
+        },
+        pointer_width: 32,
+        data_layout: "e-m:e-p:32:32-i64:64-n32-S128".into(),
+        arch: "loongarch32".into(),
+        options: TargetOptions {
+            cpu: "generic".into(),
+            features: "-f,-d".into(),
+            abi: "softfloat".into(),
+            linker_flavor: LinkerFlavor::Gnu(Cc::No, Lld::Yes),
+            linker: Some("rust-lld".into()),
+            llvm_abiname: "ilp32s".into(),
+            max_atomic_width: Some(32),
+            relocation_model: RelocModel::Static,
+            panic_strategy: PanicStrategy::Abort,
+            ..Default::default()
+        },
+    }
+}
diff --git a/compiler/rustc_target/src/spec/targets/s390x_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/targets/s390x_unknown_linux_gnu.rs
index e0d16a7bfa5..cdcf7d62a3e 100644
--- a/compiler/rustc_target/src/spec/targets/s390x_unknown_linux_gnu.rs
+++ b/compiler/rustc_target/src/spec/targets/s390x_unknown_linux_gnu.rs
@@ -1,4 +1,4 @@
-use rustc_abi::Endian;
+use rustc_abi::{Align, Endian};
 
 use crate::spec::{SanitizerSet, StackProbeType, Target, TargetMetadata, base};
 
@@ -8,7 +8,7 @@ pub(crate) fn target() -> Target {
     // z10 is the oldest CPU supported by LLVM
     base.cpu = "z10".into();
     base.max_atomic_width = Some(128);
-    base.min_global_align = Some(16);
+    base.min_global_align = Some(Align::from_bits(16).unwrap());
     base.stack_probes = StackProbeType::Inline;
     base.supported_sanitizers =
         SanitizerSet::ADDRESS | SanitizerSet::LEAK | SanitizerSet::MEMORY | SanitizerSet::THREAD;
diff --git a/compiler/rustc_target/src/spec/targets/s390x_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/targets/s390x_unknown_linux_musl.rs
index 47050c1f769..e9522ac760e 100644
--- a/compiler/rustc_target/src/spec/targets/s390x_unknown_linux_musl.rs
+++ b/compiler/rustc_target/src/spec/targets/s390x_unknown_linux_musl.rs
@@ -1,4 +1,4 @@
-use rustc_abi::Endian;
+use rustc_abi::{Align, Endian};
 
 use crate::spec::{SanitizerSet, StackProbeType, Target, TargetMetadata, base};
 
@@ -8,7 +8,7 @@ pub(crate) fn target() -> Target {
     // z10 is the oldest CPU supported by LLVM
     base.cpu = "z10".into();
     base.max_atomic_width = Some(128);
-    base.min_global_align = Some(16);
+    base.min_global_align = Some(Align::from_bits(16).unwrap());
     base.static_position_independent_executables = true;
     base.stack_probes = StackProbeType::Inline;
     base.supported_sanitizers =
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_apple_darwin.rs b/compiler/rustc_target/src/spec/targets/x86_64_apple_darwin.rs
index 2f868e38f1a..eba595ba7dd 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_apple_darwin.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_apple_darwin.rs
@@ -1,5 +1,5 @@
 use crate::spec::base::apple::{Arch, TargetAbi, base};
-use crate::spec::{FramePointer, SanitizerSet, Target, TargetMetadata, TargetOptions};
+use crate::spec::{SanitizerSet, Target, TargetMetadata, TargetOptions};
 
 pub(crate) fn target() -> Target {
     let (opts, llvm_target, arch) = base("macos", Arch::X86_64, TargetAbi::Normal);
@@ -18,11 +18,11 @@ pub(crate) fn target() -> Target {
         options: TargetOptions {
             mcount: "\u{1}mcount".into(),
             max_atomic_width: Some(128), // penryn+ supports cmpxchg16b
-            frame_pointer: FramePointer::Always,
             supported_sanitizers: SanitizerSet::ADDRESS
                 | SanitizerSet::CFI
                 | SanitizerSet::LEAK
                 | SanitizerSet::THREAD,
+            supports_xray: true,
             ..opts
         },
     }
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_unknown_uefi.rs b/compiler/rustc_target/src/spec/targets/x86_64_unknown_uefi.rs
index 07f853dacaf..0cf6a879462 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_unknown_uefi.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_unknown_uefi.rs
@@ -5,7 +5,8 @@
 // The win64 ABI is used. It differs from the sysv64 ABI, so we must use a windows target with
 // LLVM. "x86_64-unknown-windows" is used to get the minimal subset of windows-specific features.
 
-use crate::callconv::Conv;
+use rustc_abi::{CanonAbi, X86Call};
+
 use crate::spec::{RustcAbi, Target, TargetMetadata, base};
 
 pub(crate) fn target() -> Target {
@@ -13,7 +14,7 @@ pub(crate) fn target() -> Target {
     base.cpu = "x86-64".into();
     base.plt_by_default = false;
     base.max_atomic_width = Some(64);
-    base.entry_abi = Conv::X86_64Win64;
+    base.entry_abi = CanonAbi::X86(X86Call::Win64);
 
     // We disable MMX and SSE for now, even though UEFI allows using them. Problem is, you have to
     // enable these CPU features explicitly before their first use, otherwise their instructions
diff --git a/compiler/rustc_target/src/spec/targets/x86_64h_apple_darwin.rs b/compiler/rustc_target/src/spec/targets/x86_64h_apple_darwin.rs
index 11010b7d92f..e64556c4132 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64h_apple_darwin.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64h_apple_darwin.rs
@@ -1,10 +1,9 @@
 use crate::spec::base::apple::{Arch, TargetAbi, base};
-use crate::spec::{FramePointer, SanitizerSet, Target, TargetMetadata, TargetOptions};
+use crate::spec::{SanitizerSet, Target, TargetMetadata, TargetOptions};
 
 pub(crate) fn target() -> Target {
     let (mut opts, llvm_target, arch) = base("macos", Arch::X86_64h, TargetAbi::Normal);
     opts.max_atomic_width = Some(128);
-    opts.frame_pointer = FramePointer::Always;
     opts.supported_sanitizers =
         SanitizerSet::ADDRESS | SanitizerSet::CFI | SanitizerSet::LEAK | SanitizerSet::THREAD;
 
diff --git a/compiler/rustc_target/src/target_features.rs b/compiler/rustc_target/src/target_features.rs
index 99b04ac2720..c1f128fdc87 100644
--- a/compiler/rustc_target/src/target_features.rs
+++ b/compiler/rustc_target/src/target_features.rs
@@ -710,29 +710,35 @@ static LOONGARCH_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
     // tidy-alphabetical-end
 ];
 
+#[rustfmt::skip]
 const IBMZ_FEATURES: &[(&str, Stability, ImpliedFeatures)] = &[
     // tidy-alphabetical-start
     ("backchain", Unstable(sym::s390x_target_feature), &[]),
+    ("concurrent-functions", Unstable(sym::s390x_target_feature), &[]),
     ("deflate-conversion", Unstable(sym::s390x_target_feature), &[]),
     ("enhanced-sort", Unstable(sym::s390x_target_feature), &[]),
     ("guarded-storage", Unstable(sym::s390x_target_feature), &[]),
     ("high-word", Unstable(sym::s390x_target_feature), &[]),
+    // LLVM does not define message-security-assist-extension versions 1, 2, 6, 10 and 11.
+    ("message-security-assist-extension12", Unstable(sym::s390x_target_feature), &[]),
+    ("message-security-assist-extension3", Unstable(sym::s390x_target_feature), &[]),
+    ("message-security-assist-extension4", Unstable(sym::s390x_target_feature), &[]),
+    ("message-security-assist-extension5", Unstable(sym::s390x_target_feature), &[]),
+    ("message-security-assist-extension8", Unstable(sym::s390x_target_feature), &["message-security-assist-extension3"]),
+    ("message-security-assist-extension9", Unstable(sym::s390x_target_feature), &["message-security-assist-extension3", "message-security-assist-extension4"]),
+    ("miscellaneous-extensions-2", Unstable(sym::s390x_target_feature), &[]),
+    ("miscellaneous-extensions-3", Unstable(sym::s390x_target_feature), &[]),
+    ("miscellaneous-extensions-4", Unstable(sym::s390x_target_feature), &[]),
     ("nnp-assist", Unstable(sym::s390x_target_feature), &["vector"]),
     ("transactional-execution", Unstable(sym::s390x_target_feature), &[]),
     ("vector", Unstable(sym::s390x_target_feature), &[]),
     ("vector-enhancements-1", Unstable(sym::s390x_target_feature), &["vector"]),
     ("vector-enhancements-2", Unstable(sym::s390x_target_feature), &["vector-enhancements-1"]),
+    ("vector-enhancements-3", Unstable(sym::s390x_target_feature), &["vector-enhancements-2"]),
     ("vector-packed-decimal", Unstable(sym::s390x_target_feature), &["vector"]),
-    (
-        "vector-packed-decimal-enhancement",
-        Unstable(sym::s390x_target_feature),
-        &["vector-packed-decimal"],
-    ),
-    (
-        "vector-packed-decimal-enhancement-2",
-        Unstable(sym::s390x_target_feature),
-        &["vector-packed-decimal-enhancement"],
-    ),
+    ("vector-packed-decimal-enhancement", Unstable(sym::s390x_target_feature), &["vector-packed-decimal"]),
+    ("vector-packed-decimal-enhancement-2", Unstable(sym::s390x_target_feature), &["vector-packed-decimal-enhancement"]),
+    ("vector-packed-decimal-enhancement-3", Unstable(sym::s390x_target_feature), &["vector-packed-decimal-enhancement-2"]),
     // tidy-alphabetical-end
 ];
 
@@ -840,7 +846,7 @@ impl Target {
             "wasm32" | "wasm64" => WASM_FEATURES,
             "bpf" => BPF_FEATURES,
             "csky" => CSKY_FEATURES,
-            "loongarch64" => LOONGARCH_FEATURES,
+            "loongarch32" | "loongarch64" => LOONGARCH_FEATURES,
             "s390x" => IBMZ_FEATURES,
             "sparc" | "sparc64" => SPARC_FEATURES,
             "m68k" => M68K_FEATURES,
@@ -854,7 +860,7 @@ impl Target {
             "aarch64" | "arm64ec" => AARCH64_FEATURES_FOR_CORRECT_VECTOR_ABI,
             "arm" => ARM_FEATURES_FOR_CORRECT_VECTOR_ABI,
             "powerpc" | "powerpc64" => POWERPC_FEATURES_FOR_CORRECT_VECTOR_ABI,
-            "loongarch64" => LOONGARCH_FEATURES_FOR_CORRECT_VECTOR_ABI,
+            "loongarch32" | "loongarch64" => LOONGARCH_FEATURES_FOR_CORRECT_VECTOR_ABI,
             "riscv32" | "riscv64" => RISCV_FEATURES_FOR_CORRECT_VECTOR_ABI,
             "wasm32" | "wasm64" => WASM_FEATURES_FOR_CORRECT_VECTOR_ABI,
             "s390x" => S390X_FEATURES_FOR_CORRECT_VECTOR_ABI,
@@ -980,14 +986,16 @@ impl Target {
                 // the use of soft-float, so all we can do here is some crude hacks.
                 match &*self.abi {
                     "softfloat" => {
-                        // This is not fully correct, LLVM actually doesn't let us enforce the softfloat
-                        // ABI properly... see <https://github.com/rust-lang/rust/issues/134375>.
-                        // FIXME: should we forbid "neon" here? But that would be a breaking change.
-                        NOTHING
+                        // LLVM will use float registers when `fp-armv8` is available, e.g. for
+                        // calls to built-ins. The only way to ensure a consistent softfloat ABI
+                        // on aarch64 is to never enable `fp-armv8`, so we enforce that.
+                        // In Rust we tie `neon` and `fp-armv8` together, therefore `neon` is the
+                        // feature we have to mark as incompatible.
+                        FeatureConstraints { required: &[], incompatible: &["neon"] }
                     }
                     _ => {
                         // Everything else is assumed to use a hardfloat ABI. neon and fp-armv8 must be enabled.
-                        // These are Rust feature names and we use "neon" to control both of them.
+                        // `FeatureConstraints` uses Rust feature names, hence only "neon" shows up.
                         FeatureConstraints { required: &["neon"], incompatible: &[] }
                     }
                 }
@@ -1026,7 +1034,7 @@ impl Target {
                     _ => unreachable!(),
                 }
             }
-            "loongarch64" => {
+            "loongarch32" | "loongarch64" => {
                 // LoongArch handles ABI in a very sane way, being fully explicit via `llvm_abiname`
                 // about what the intended ABI is.
                 match &*self.llvm_abiname {
diff --git a/compiler/rustc_trait_selection/messages.ftl b/compiler/rustc_trait_selection/messages.ftl
index 00922c6038e..8232da4df43 100644
--- a/compiler/rustc_trait_selection/messages.ftl
+++ b/compiler/rustc_trait_selection/messages.ftl
@@ -72,8 +72,6 @@ trait_selection_adjust_signature_remove_borrow = consider adjusting the signatur
 
 trait_selection_ascribe_user_type_prove_predicate = ...so that the where clause holds
 
-trait_selection_async_closure_not_fn = async closure does not implement `{$kind}` because it captures state from its environment
-
 trait_selection_await_both_futures = consider `await`ing on both `Future`s
 trait_selection_await_future = consider `await`ing on the `Future`
 trait_selection_await_note = calling an async function returns a future
@@ -123,6 +121,8 @@ trait_selection_closure_kind_requirement = the requirement to implement `{$trait
 
 trait_selection_compare_impl_item_obligation = ...so that the definition in impl matches the definition from the trait
 trait_selection_consider_specifying_length = consider specifying the actual array length
+trait_selection_coro_closure_not_fn = {$coro_kind}closure does not implement `{$kind}` because it captures state from its environment
+
 trait_selection_data_flows = ...but data{$label_var1_exists ->
     [true] {" "}from `{$label_var1}`
     *[false] {""}
@@ -337,6 +337,8 @@ trait_selection_rustc_on_unimplemented_expected_one_predicate_in_not = expected
     .label = unexpected quantity of predicates here
 trait_selection_rustc_on_unimplemented_invalid_flag = invalid flag in `on`-clause
     .label = expected one of the `crate_local`, `direct` or `from_desugaring` flags, not `{$invalid_flag}`
+trait_selection_rustc_on_unimplemented_invalid_name = invalid name in `on`-clause
+    .label = expected one of `cause`, `from_desugaring`, `Self` or any generic parameter of the trait, not `{$invalid_name}`
 trait_selection_rustc_on_unimplemented_invalid_predicate = this predicate is invalid
     .label = expected one of `any`, `all` or `not` here, not `{$invalid_pred}`
 trait_selection_rustc_on_unimplemented_missing_value = this attribute must have a value
diff --git a/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs b/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs
index fdd547448f0..2c16672d786 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs
@@ -73,7 +73,7 @@ use rustc_middle::ty::{
     TypeVisitableExt,
 };
 use rustc_span::def_id::LOCAL_CRATE;
-use rustc_span::{BytePos, DesugaringKind, Pos, Span, sym};
+use rustc_span::{BytePos, DUMMY_SP, DesugaringKind, Pos, Span, sym};
 use tracing::{debug, instrument};
 
 use crate::error_reporting::TypeErrCtxt;
@@ -194,7 +194,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
             _ => return None,
         };
 
-        let future_trait = self.tcx.require_lang_item(LangItem::Future, None);
+        let future_trait = self.tcx.require_lang_item(LangItem::Future, DUMMY_SP);
         let item_def_id = self.tcx.associated_item_def_ids(future_trait)[0];
 
         self.tcx
@@ -738,7 +738,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
                 value.push_normal(", ");
             }
 
-            match arg.unpack() {
+            match arg.kind() {
                 ty::GenericArgKind::Lifetime(lt) => {
                     let s = lt.to_string();
                     value.push_normal(if s.is_empty() { "'_" } else { &s });
@@ -1166,7 +1166,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
 
                     for (i, (arg1, arg2)) in sub1.iter().zip(sub2).enumerate().take(len) {
                         self.push_comma(&mut values.0, &mut values.1, i);
-                        match arg1.unpack() {
+                        match arg1.kind() {
                             // At one point we'd like to elide all lifetimes here, they are
                             // irrelevant for all diagnostics that use this output.
                             //
@@ -1509,7 +1509,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
                 }
                 let (is_simple_error, exp_found) = match values {
                     ValuePairs::Terms(ExpectedFound { expected, found }) => {
-                        match (expected.unpack(), found.unpack()) {
+                        match (expected.kind(), found.kind()) {
                             (ty::TermKind::Ty(expected), ty::TermKind::Ty(found)) => {
                                 let is_simple_err =
                                     expected.is_simple_text() && found.is_simple_text();
@@ -2026,7 +2026,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
                 }
                 LetVisitor { span }.visit_body(body).break_value()
             }
-            hir::Node::Item(hir::Item { kind: hir::ItemKind::Const(_, ty, _, _), .. }) => {
+            hir::Node::Item(hir::Item { kind: hir::ItemKind::Const(_, _, ty, _), .. }) => {
                 Some(&ty.peel_refs().kind)
             }
             _ => None,
@@ -2156,7 +2156,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
             return None;
         }
 
-        Some(match (exp_found.expected.unpack(), exp_found.found.unpack()) {
+        Some(match (exp_found.expected.kind(), exp_found.found.kind()) {
             (ty::TermKind::Ty(expected), ty::TermKind::Ty(found)) => {
                 let (mut exp, mut fnd) = self.cmp(expected, found);
                 // Use the terminal width as the basis to determine when to compress the printed
diff --git a/compiler/rustc_trait_selection/src/error_reporting/infer/need_type_info.rs b/compiler/rustc_trait_selection/src/error_reporting/infer/need_type_info.rs
index cb1c9c75369..bfef3340b32 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/infer/need_type_info.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/infer/need_type_info.rs
@@ -215,7 +215,7 @@ impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for ClosureEraser<'a, 'tcx> {
                                 // `_` because then we'd end up with `Vec<_, _>`, instead of
                                 // `Vec<_>`.
                                 arg
-                            } else if let GenericArgKind::Type(_) = arg.unpack() {
+                            } else if let GenericArgKind::Type(_) = arg.kind() {
                                 // We don't replace lifetime or const params, only type params.
                                 self.new_infer().into()
                             } else {
@@ -347,7 +347,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
         highlight: ty::print::RegionHighlightMode<'tcx>,
     ) -> InferenceDiagnosticsData {
         let tcx = self.tcx;
-        match term.unpack() {
+        match term.kind() {
             TermKind::Ty(ty) => {
                 if let ty::Infer(ty::TyVar(ty_vid)) = *ty.kind() {
                     let var_origin = self.infcx.type_var_origin(ty_vid);
@@ -568,7 +568,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
                                 return arg;
                             }
 
-                            match arg.unpack() {
+                            match arg.kind() {
                                 GenericArgKind::Lifetime(_) => bug!("unexpected lifetime"),
                                 GenericArgKind::Type(_) => self.next_ty_var(DUMMY_SP).into(),
                                 GenericArgKind::Const(_) => self.next_const_var(DUMMY_SP).into(),
@@ -803,7 +803,7 @@ impl<'a, 'tcx> FindInferSourceVisitor<'a, 'tcx> {
         }
         impl<'tcx> CostCtxt<'tcx> {
             fn arg_cost(self, arg: GenericArg<'tcx>) -> usize {
-                match arg.unpack() {
+                match arg.kind() {
                     GenericArgKind::Lifetime(_) => 0, // erased
                     GenericArgKind::Type(ty) => self.ty_cost(ty),
                     GenericArgKind::Const(_) => 3, // some non-zero value
@@ -898,7 +898,7 @@ impl<'a, 'tcx> FindInferSourceVisitor<'a, 'tcx> {
             return true;
         }
 
-        match (arg.unpack(), self.target.unpack()) {
+        match (arg.kind(), self.target.kind()) {
             (GenericArgKind::Type(inner_ty), TermKind::Ty(target_ty)) => {
                 use ty::{Infer, TyVar};
                 match (inner_ty.kind(), target_ty.kind()) {
@@ -929,7 +929,7 @@ impl<'a, 'tcx> FindInferSourceVisitor<'a, 'tcx> {
             if self.generic_arg_is_target(inner) {
                 return true;
             }
-            match inner.unpack() {
+            match inner.kind() {
                 GenericArgKind::Lifetime(_) => {}
                 GenericArgKind::Type(ty) => {
                     if matches!(
diff --git a/compiler/rustc_trait_selection/src/error_reporting/infer/suggest.rs b/compiler/rustc_trait_selection/src/error_reporting/infer/suggest.rs
index cdbb92f4c7b..3804c13acce 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/infer/suggest.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/infer/suggest.rs
@@ -664,8 +664,8 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         let Some(found) = exp_found.found.args.get(1) else {
             return;
         };
-        let expected = expected.unpack();
-        let found = found.unpack();
+        let expected = expected.kind();
+        let found = found.kind();
         // 3. Extract the tuple type from Fn trait and suggest the change.
         if let GenericArgKind::Type(expected) = expected
             && let GenericArgKind::Type(found) = found
diff --git a/compiler/rustc_trait_selection/src/error_reporting/traits/call_kind.rs b/compiler/rustc_trait_selection/src/error_reporting/traits/call_kind.rs
index d8b405e904c..8a67e4ccd45 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/traits/call_kind.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/traits/call_kind.rs
@@ -6,7 +6,7 @@ use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
 use rustc_hir::{LangItem, lang_items};
 use rustc_middle::ty::{AssocItemContainer, GenericArgsRef, Instance, Ty, TyCtxt, TypingEnv};
-use rustc_span::{DesugaringKind, Ident, Span, sym};
+use rustc_span::{DUMMY_SP, DesugaringKind, Ident, Span, sym};
 use tracing::debug;
 
 use crate::traits::specialization_graph;
@@ -31,9 +31,9 @@ impl CallDesugaringKind {
     pub fn trait_def_id(self, tcx: TyCtxt<'_>) -> DefId {
         match self {
             Self::ForLoopIntoIter => tcx.get_diagnostic_item(sym::IntoIterator).unwrap(),
-            Self::ForLoopNext => tcx.require_lang_item(LangItem::Iterator, None),
+            Self::ForLoopNext => tcx.require_lang_item(LangItem::Iterator, DUMMY_SP),
             Self::QuestionBranch | Self::TryBlockFromOutput => {
-                tcx.require_lang_item(LangItem::Try, None)
+                tcx.require_lang_item(LangItem::Try, DUMMY_SP)
             }
             Self::QuestionFromResidual => tcx.get_diagnostic_item(sym::FromResidual).unwrap(),
             Self::Await => tcx.get_diagnostic_item(sym::IntoFuture).unwrap(),
diff --git a/compiler/rustc_trait_selection/src/error_reporting/traits/fulfillment_errors.rs b/compiler/rustc_trait_selection/src/error_reporting/traits/fulfillment_errors.rs
index 970160ba212..fc5be111144 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/traits/fulfillment_errors.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/traits/fulfillment_errors.rs
@@ -42,9 +42,7 @@ use super::{
 use crate::error_reporting::TypeErrCtxt;
 use crate::error_reporting::infer::TyCategory;
 use crate::error_reporting::traits::report_dyn_incompatibility;
-use crate::errors::{
-    AsyncClosureNotFn, ClosureFnMutLabel, ClosureFnOnceLabel, ClosureKindMismatch,
-};
+use crate::errors::{ClosureFnMutLabel, ClosureFnOnceLabel, ClosureKindMismatch, CoroClosureNotFn};
 use crate::infer::{self, InferCtxt, InferCtxtExt as _};
 use crate::traits::query::evaluate_obligation::InferCtxtExt as _;
 use crate::traits::{
@@ -841,16 +839,17 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
                 return None;
             };
 
-        let (closure_def_id, found_args, by_ref_captures) = match *self_ty.kind() {
+        let (closure_def_id, found_args, has_self_borrows) = match *self_ty.kind() {
             ty::Closure(def_id, args) => {
-                (def_id, args.as_closure().sig().map_bound(|sig| sig.inputs()[0]), None)
+                (def_id, args.as_closure().sig().map_bound(|sig| sig.inputs()[0]), false)
             }
             ty::CoroutineClosure(def_id, args) => (
                 def_id,
                 args.as_coroutine_closure()
                     .coroutine_closure_sig()
                     .map_bound(|sig| sig.tupled_inputs_ty),
-                Some(args.as_coroutine_closure().coroutine_captures_by_ref_ty()),
+                !args.as_coroutine_closure().tupled_upvars_ty().is_ty_var()
+                    && args.as_coroutine_closure().has_self_borrows(),
             ),
             _ => return None,
         };
@@ -884,13 +883,19 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
         // If the closure has captures, then perhaps the reason that the trait
         // is unimplemented is because async closures don't implement `Fn`/`FnMut`
         // if they have captures.
-        if let Some(by_ref_captures) = by_ref_captures
-            && let ty::FnPtr(sig_tys, _) = by_ref_captures.kind()
-            && !sig_tys.skip_binder().output().is_unit()
-        {
-            let mut err = self.dcx().create_err(AsyncClosureNotFn {
+        if has_self_borrows && expected_kind != ty::ClosureKind::FnOnce {
+            let coro_kind = match self
+                .tcx
+                .coroutine_kind(self.tcx.coroutine_for_closure(closure_def_id))
+                .unwrap()
+            {
+                rustc_hir::CoroutineKind::Desugared(desugaring, _) => desugaring.to_string(),
+                coro => coro.to_string(),
+            };
+            let mut err = self.dcx().create_err(CoroClosureNotFn {
                 span: self.tcx.def_span(closure_def_id),
                 kind: expected_kind.as_str(),
+                coro_kind,
             });
             self.note_obligation_cause(&mut err, &obligation);
             return Some(err.emit());
@@ -1503,11 +1508,11 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
                     return None;
                 };
 
-                let Ok(Some(ImplSource::UserDefined(impl_data))) = SelectionContext::new(self)
-                    .poly_select(&obligation.with(
-                        self.tcx,
-                        predicate.kind().rebind(proj.projection_term.trait_ref(self.tcx)),
-                    ))
+                let trait_ref = self.enter_forall_and_leak_universe(
+                    predicate.kind().rebind(proj.projection_term.trait_ref(self.tcx)),
+                );
+                let Ok(Some(ImplSource::UserDefined(impl_data))) =
+                    SelectionContext::new(self).select(&obligation.with(self.tcx, trait_ref))
                 else {
                     return None;
                 };
@@ -2448,7 +2453,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
             }
             if let ty::Adt(def, args) = self_ty.kind()
                 && let [arg] = &args[..]
-                && let ty::GenericArgKind::Type(ty) = arg.unpack()
+                && let ty::GenericArgKind::Type(ty) = arg.kind()
                 && let ty::Adt(inner_def, _) = ty.kind()
                 && inner_def == def
             {
diff --git a/compiler/rustc_trait_selection/src/error_reporting/traits/on_unimplemented.rs b/compiler/rustc_trait_selection/src/error_reporting/traits/on_unimplemented.rs
index d5ee6e2123a..37968386e9a 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/traits/on_unimplemented.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/traits/on_unimplemented.rs
@@ -429,7 +429,19 @@ impl<'tcx> OnUnimplementedDirective {
                 .next()
                 .ok_or_else(|| tcx.dcx().emit_err(InvalidOnClause::Empty { span }))?;
 
-            match OnUnimplementedCondition::parse(cond) {
+            let generics: Vec<Symbol> = tcx
+                .generics_of(item_def_id)
+                .own_params
+                .iter()
+                .filter_map(|param| {
+                    if matches!(param.kind, GenericParamDefKind::Lifetime) {
+                        None
+                    } else {
+                        Some(param.name)
+                    }
+                })
+                .collect();
+            match OnUnimplementedCondition::parse(cond, &generics) {
                 Ok(condition) => Some(condition),
                 Err(e) => return Err(tcx.dcx().emit_err(e)),
             }
diff --git a/compiler/rustc_trait_selection/src/error_reporting/traits/on_unimplemented_condition.rs b/compiler/rustc_trait_selection/src/error_reporting/traits/on_unimplemented_condition.rs
index 13753761f09..e8ea9f2d23e 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/traits/on_unimplemented_condition.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/traits/on_unimplemented_condition.rs
@@ -26,9 +26,12 @@ impl OnUnimplementedCondition {
         })
     }
 
-    pub(crate) fn parse(input: &MetaItemInner) -> Result<Self, InvalidOnClause> {
+    pub(crate) fn parse(
+        input: &MetaItemInner,
+        generics: &[Symbol],
+    ) -> Result<Self, InvalidOnClause> {
         let span = input.span();
-        let pred = Predicate::parse(input)?;
+        let pred = Predicate::parse(input, generics)?;
         Ok(OnUnimplementedCondition { span, pred })
     }
 }
@@ -52,7 +55,7 @@ enum Predicate {
 }
 
 impl Predicate {
-    fn parse(input: &MetaItemInner) -> Result<Self, InvalidOnClause> {
+    fn parse(input: &MetaItemInner, generics: &[Symbol]) -> Result<Self, InvalidOnClause> {
         let meta_item = match input {
             MetaItemInner::MetaItem(meta_item) => meta_item,
             MetaItemInner::Lit(lit) => {
@@ -69,10 +72,10 @@ impl Predicate {
 
         match meta_item.kind {
             MetaItemKind::List(ref mis) => match predicate.name {
-                sym::any => Ok(Predicate::Any(Predicate::parse_sequence(mis)?)),
-                sym::all => Ok(Predicate::All(Predicate::parse_sequence(mis)?)),
+                sym::any => Ok(Predicate::Any(Predicate::parse_sequence(mis, generics)?)),
+                sym::all => Ok(Predicate::All(Predicate::parse_sequence(mis, generics)?)),
                 sym::not => match &**mis {
-                    [one] => Ok(Predicate::Not(Box::new(Predicate::parse(one)?))),
+                    [one] => Ok(Predicate::Not(Box::new(Predicate::parse(one, generics)?))),
                     [first, .., last] => Err(InvalidOnClause::ExpectedOnePredInNot {
                         span: first.span().to(last.span()),
                     }),
@@ -83,7 +86,7 @@ impl Predicate {
                 }
             },
             MetaItemKind::NameValue(MetaItemLit { symbol, .. }) => {
-                let name = Name::parse(predicate);
+                let name = Name::parse(predicate, generics)?;
                 let value = FilterFormatString::parse(symbol);
                 let kv = NameValue { name, value };
                 Ok(Predicate::Match(kv))
@@ -95,8 +98,11 @@ impl Predicate {
         }
     }
 
-    fn parse_sequence(sequence: &[MetaItemInner]) -> Result<Vec<Self>, InvalidOnClause> {
-        sequence.iter().map(Predicate::parse).collect()
+    fn parse_sequence(
+        sequence: &[MetaItemInner],
+        generics: &[Symbol],
+    ) -> Result<Vec<Self>, InvalidOnClause> {
+        sequence.iter().map(|item| Predicate::parse(item, generics)).collect()
     }
 
     fn eval(&self, eval: &mut impl FnMut(FlagOrNv<'_>) -> bool) -> bool {
@@ -156,14 +162,13 @@ enum Name {
 }
 
 impl Name {
-    fn parse(Ident { name, .. }: Ident) -> Self {
+    fn parse(Ident { name, span }: Ident, generics: &[Symbol]) -> Result<Self, InvalidOnClause> {
         match name {
-            sym::_Self | kw::SelfUpper => Name::SelfUpper,
-            sym::from_desugaring => Name::FromDesugaring,
-            sym::cause => Name::Cause,
-            // FIXME(mejrs) Perhaps we should start checking that
-            // this actually is a valid generic parameter?
-            generic => Name::GenericArg(generic),
+            kw::SelfUpper => Ok(Name::SelfUpper),
+            sym::from_desugaring => Ok(Name::FromDesugaring),
+            sym::cause => Ok(Name::Cause),
+            generic if generics.contains(&generic) => Ok(Name::GenericArg(generic)),
+            invalid_name => Err(InvalidOnClause::InvalidName { invalid_name, span }),
         }
     }
 }
diff --git a/compiler/rustc_trait_selection/src/error_reporting/traits/on_unimplemented_format.rs b/compiler/rustc_trait_selection/src/error_reporting/traits/on_unimplemented_format.rs
index ce170f820e1..7c1dfc1728f 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/traits/on_unimplemented_format.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/traits/on_unimplemented_format.rs
@@ -2,8 +2,8 @@ use std::fmt;
 use std::ops::Range;
 
 use errors::*;
-use rustc_middle::ty::TyCtxt;
 use rustc_middle::ty::print::TraitRefPrintSugared;
+use rustc_middle::ty::{GenericParamDefKind, TyCtxt};
 use rustc_parse_format::{
     Alignment, Argument, Count, FormatSpec, ParseError, ParseMode, Parser, Piece as RpfPiece,
     Position,
@@ -232,48 +232,16 @@ fn parse_arg<'tcx>(
 ) -> FormatArg {
     let (Ctx::RustcOnUnimplemented { tcx, trait_def_id }
     | Ctx::DiagnosticOnUnimplemented { tcx, trait_def_id }) = ctx;
-    let trait_name = tcx.item_ident(*trait_def_id);
-    let generics = tcx.generics_of(trait_def_id);
+
     let span = slice_span(input_span, arg.position_span.clone());
 
     match arg.position {
         // Something like "hello {name}"
         Position::ArgumentNamed(name) => match (ctx, Symbol::intern(name)) {
-            // accepted, but deprecated
-            (Ctx::RustcOnUnimplemented { .. }, sym::_Self) => {
-                warnings
-                    .push(FormatWarning::FutureIncompat { span, help: String::from("use {Self}") });
-                FormatArg::SelfUpper
-            }
-            (
-                Ctx::RustcOnUnimplemented { .. },
-                sym::from_desugaring
-                | sym::crate_local
-                | sym::direct
-                | sym::cause
-                | sym::float
-                | sym::integer_
-                | sym::integral,
-            ) => {
-                warnings.push(FormatWarning::FutureIncompat {
-                    span,
-                    help: String::from("don't use this in a format string"),
-                });
-                FormatArg::AsIs(String::new())
-            }
-
             // Only `#[rustc_on_unimplemented]` can use these
             (Ctx::RustcOnUnimplemented { .. }, sym::ItemContext) => FormatArg::ItemContext,
             (Ctx::RustcOnUnimplemented { .. }, sym::This) => FormatArg::This,
             (Ctx::RustcOnUnimplemented { .. }, sym::Trait) => FormatArg::Trait,
-            // `{ThisTraitsName}`. Some attrs in std use this, but I'd like to change it to the more general `{This}`
-            // because that'll be simpler to parse and extend in the future
-            (Ctx::RustcOnUnimplemented { .. }, name) if name == trait_name.name => {
-                warnings
-                    .push(FormatWarning::FutureIncompat { span, help: String::from("use {This}") });
-                FormatArg::This
-            }
-
             // Any attribute can use these
             (
                 Ctx::RustcOnUnimplemented { .. } | Ctx::DiagnosticOnUnimplemented { .. },
@@ -282,7 +250,10 @@ fn parse_arg<'tcx>(
             (
                 Ctx::RustcOnUnimplemented { .. } | Ctx::DiagnosticOnUnimplemented { .. },
                 generic_param,
-            ) if generics.own_params.iter().any(|param| param.name == generic_param) => {
+            ) if tcx.generics_of(trait_def_id).own_params.iter().any(|param| {
+                !matches!(param.kind, GenericParamDefKind::Lifetime) && param.name == generic_param
+            }) =>
+            {
                 FormatArg::GenericParam { generic_param }
             }
 
@@ -375,39 +346,4 @@ pub mod errors {
     #[diag(trait_selection_missing_options_for_on_unimplemented_attr)]
     #[help]
     pub struct MissingOptionsForOnUnimplementedAttr;
-
-    #[derive(LintDiagnostic)]
-    #[diag(trait_selection_ignored_diagnostic_option)]
-    pub struct IgnoredDiagnosticOption {
-        pub option_name: &'static str,
-        #[label]
-        pub span: Span,
-        #[label(trait_selection_other_label)]
-        pub prev_span: Span,
-    }
-
-    impl IgnoredDiagnosticOption {
-        pub fn maybe_emit_warning<'tcx>(
-            tcx: TyCtxt<'tcx>,
-            item_def_id: DefId,
-            new: Option<Span>,
-            old: Option<Span>,
-            option_name: &'static str,
-        ) {
-            if let (Some(new_item), Some(old_item)) = (new, old) {
-                if let Some(item_def_id) = item_def_id.as_local() {
-                    tcx.emit_node_span_lint(
-                        UNKNOWN_OR_MALFORMED_DIAGNOSTIC_ATTRIBUTES,
-                        tcx.local_def_id_to_hir_id(item_def_id),
-                        new_item,
-                        IgnoredDiagnosticOption {
-                            span: new_item,
-                            prev_span: old_item,
-                            option_name,
-                        },
-                    );
-                }
-            }
-        }
-    }
 }
diff --git a/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs b/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs
index 8801397b775..68bd9440538 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/traits/suggestions.rs
@@ -351,14 +351,14 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
 
                 hir::Node::Item(hir::Item {
                     kind:
-                        hir::ItemKind::Struct(_, _, generics)
-                        | hir::ItemKind::Enum(_, _, generics)
-                        | hir::ItemKind::Union(_, _, generics)
+                        hir::ItemKind::Struct(_, generics, _)
+                        | hir::ItemKind::Enum(_, generics, _)
+                        | hir::ItemKind::Union(_, generics, _)
                         | hir::ItemKind::Trait(_, _, _, generics, ..)
                         | hir::ItemKind::Impl(hir::Impl { generics, .. })
                         | hir::ItemKind::Fn { generics, .. }
-                        | hir::ItemKind::TyAlias(_, _, generics)
-                        | hir::ItemKind::Const(_, _, generics, _)
+                        | hir::ItemKind::TyAlias(_, generics, _)
+                        | hir::ItemKind::Const(_, generics, _, _)
                         | hir::ItemKind::TraitAlias(_, generics, _),
                     ..
                 })
@@ -411,14 +411,14 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
 
                 hir::Node::Item(hir::Item {
                     kind:
-                        hir::ItemKind::Struct(_, _, generics)
-                        | hir::ItemKind::Enum(_, _, generics)
-                        | hir::ItemKind::Union(_, _, generics)
+                        hir::ItemKind::Struct(_, generics, _)
+                        | hir::ItemKind::Enum(_, generics, _)
+                        | hir::ItemKind::Union(_, generics, _)
                         | hir::ItemKind::Trait(_, _, _, generics, ..)
                         | hir::ItemKind::Impl(hir::Impl { generics, .. })
                         | hir::ItemKind::Fn { generics, .. }
-                        | hir::ItemKind::TyAlias(_, _, generics)
-                        | hir::ItemKind::Const(_, _, generics, _)
+                        | hir::ItemKind::TyAlias(_, generics, _)
+                        | hir::ItemKind::Const(_, generics, _, _)
                         | hir::ItemKind::TraitAlias(_, generics, _),
                     ..
                 }) if !param_ty => {
@@ -955,7 +955,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
                 return false;
             };
 
-            let clone_trait = self.tcx.require_lang_item(LangItem::Clone, None);
+            let clone_trait = self.tcx.require_lang_item(LangItem::Clone, obligation.cause.span);
             let has_clone = |ty| {
                 self.type_implements_trait(clone_trait, [ty], obligation.param_env)
                     .must_apply_modulo_regions()
@@ -1411,7 +1411,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
             }
         }
 
-        err.span_suggestion(
+        err.span_suggestion_verbose(
             obligation.cause.span.shrink_to_lo(),
             format!(
                 "consider borrowing the value, since `&{self_ty}` can be coerced into `{target_ty}`"
@@ -1574,7 +1574,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
                     .span_extend_while_whitespace(expr_span)
                     .shrink_to_hi()
                     .to(await_expr.span.shrink_to_hi());
-                err.span_suggestion(
+                err.span_suggestion_verbose(
                     removal_span,
                     "remove the `.await`",
                     "",
@@ -2124,16 +2124,19 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
                         accessed through a specific `impl`",
                     self.tcx.def_kind_descr(assoc_item.as_def_kind(), item_def_id)
                 ));
-                err.span_suggestion(
-                    span,
-                    "use the fully qualified path to an implementation",
-                    format!(
-                        "<Type as {}>::{}",
-                        self.tcx.def_path_str(trait_ref),
-                        assoc_item.name()
-                    ),
-                    Applicability::HasPlaceholders,
-                );
+
+                if !assoc_item.is_impl_trait_in_trait() {
+                    err.span_suggestion_verbose(
+                        span,
+                        "use the fully qualified path to an implementation",
+                        format!(
+                            "<Type as {}>::{}",
+                            self.tcx.def_path_str(trait_ref),
+                            assoc_item.name()
+                        ),
+                        Applicability::HasPlaceholders,
+                    );
+                }
             }
         }
     }
@@ -2921,12 +2924,14 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
                 );
                 let sm = tcx.sess.source_map();
                 if matches!(is_constable, IsConstable::Fn | IsConstable::Ctor)
-                    && let Ok(snip) = sm.span_to_snippet(elt_span)
+                    && let Ok(_) = sm.span_to_snippet(elt_span)
                 {
-                    err.span_suggestion(
-                        elt_span,
+                    err.multipart_suggestion(
                         "create an inline `const` block",
-                        format!("const {{ {snip} }}"),
+                        vec![
+                            (elt_span.shrink_to_lo(), "const { ".to_string()),
+                            (elt_span.shrink_to_hi(), " }".to_string()),
+                        ],
                         Applicability::MachineApplicable,
                     );
                 } else {
@@ -3124,13 +3129,13 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
                     }
                 }
                 err.help("change the field's type to have a statically known size");
-                err.span_suggestion(
+                err.span_suggestion_verbose(
                     span.shrink_to_lo(),
                     "borrowed types always have a statically known size",
                     "&",
                     Applicability::MachineApplicable,
                 );
-                err.multipart_suggestion(
+                err.multipart_suggestion_verbose(
                     "the `Box` type always has a statically known size and allocates its contents \
                      in the heap",
                     vec![
@@ -3622,7 +3627,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
         trait_pred: ty::PolyTraitPredicate<'tcx>,
         span: Span,
     ) {
-        let future_trait = self.tcx.require_lang_item(LangItem::Future, None);
+        let future_trait = self.tcx.require_lang_item(LangItem::Future, span);
 
         let self_ty = self.resolve_vars_if_possible(trait_pred.self_ty());
         let impls_future = self.type_implements_trait(
@@ -3838,7 +3843,9 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
                     .expr_ty_adjusted_opt(inner_expr)
                     .unwrap_or(Ty::new_misc_error(tcx));
                 let span = inner_expr.span;
-                if Some(span) != err.span.primary_span() {
+                if Some(span) != err.span.primary_span()
+                    && !span.in_external_macro(tcx.sess.source_map())
+                {
                     err.span_label(
                         span,
                         if ty.references_error() {
@@ -4136,7 +4143,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
                         let pred = ty::Binder::dummy(ty::TraitPredicate {
                             trait_ref: ty::TraitRef::new(
                                 tcx,
-                                tcx.require_lang_item(LangItem::Clone, Some(span)),
+                                tcx.require_lang_item(LangItem::Clone, span),
                                 [*ty],
                             ),
                             polarity: ty::PredicatePolarity::Positive,
diff --git a/compiler/rustc_trait_selection/src/errors.rs b/compiler/rustc_trait_selection/src/errors.rs
index 8ab4d795c45..06f81ac554e 100644
--- a/compiler/rustc_trait_selection/src/errors.rs
+++ b/compiler/rustc_trait_selection/src/errors.rs
@@ -72,6 +72,13 @@ pub enum InvalidOnClause {
         span: Span,
         invalid_flag: Symbol,
     },
+    #[diag(trait_selection_rustc_on_unimplemented_invalid_name, code = E0232)]
+    InvalidName {
+        #[primary_span]
+        #[label]
+        span: Span,
+        invalid_name: Symbol,
+    },
 }
 
 #[derive(Diagnostic)]
@@ -194,11 +201,12 @@ pub struct ClosureFnMutLabel {
 }
 
 #[derive(Diagnostic)]
-#[diag(trait_selection_async_closure_not_fn)]
-pub(crate) struct AsyncClosureNotFn {
+#[diag(trait_selection_coro_closure_not_fn)]
+pub(crate) struct CoroClosureNotFn {
     #[primary_span]
     pub span: Span,
     pub kind: &'static str,
+    pub coro_kind: String,
 }
 
 #[derive(Diagnostic)]
@@ -585,7 +593,7 @@ impl Subdiagnostic for AddLifetimeParamsSuggestion<'_> {
                                         matches!(
                                             arg,
                                             hir::GenericArg::Lifetime(lifetime)
-                                                if lifetime.is_syntactically_hidden()
+                                                if lifetime.is_implicit()
                                         )
                                     }) {
                                         self.suggestions.push((
diff --git a/compiler/rustc_trait_selection/src/infer.rs b/compiler/rustc_trait_selection/src/infer.rs
index 0dab3adadb0..0118321befb 100644
--- a/compiler/rustc_trait_selection/src/infer.rs
+++ b/compiler/rustc_trait_selection/src/infer.rs
@@ -38,7 +38,7 @@ impl<'tcx> InferCtxt<'tcx> {
             return self.tcx.type_is_copy_modulo_regions(self.typing_env(param_env), ty);
         }
 
-        let copy_def_id = self.tcx.require_lang_item(LangItem::Copy, None);
+        let copy_def_id = self.tcx.require_lang_item(LangItem::Copy, DUMMY_SP);
 
         // This can get called from typeck (by euv), and `moves_by_default`
         // rightly refuses to work with inference variables, but
@@ -49,7 +49,7 @@ impl<'tcx> InferCtxt<'tcx> {
 
     fn type_is_clone_modulo_regions(&self, param_env: ty::ParamEnv<'tcx>, ty: Ty<'tcx>) -> bool {
         let ty = self.resolve_vars_if_possible(ty);
-        let clone_def_id = self.tcx.require_lang_item(LangItem::Clone, None);
+        let clone_def_id = self.tcx.require_lang_item(LangItem::Clone, DUMMY_SP);
         traits::type_known_to_meet_bound_modulo_regions(self, param_env, ty, clone_def_id)
     }
 
@@ -59,12 +59,12 @@ impl<'tcx> InferCtxt<'tcx> {
         ty: Ty<'tcx>,
     ) -> bool {
         let ty = self.resolve_vars_if_possible(ty);
-        let use_cloned_def_id = self.tcx.require_lang_item(LangItem::UseCloned, None);
+        let use_cloned_def_id = self.tcx.require_lang_item(LangItem::UseCloned, DUMMY_SP);
         traits::type_known_to_meet_bound_modulo_regions(self, param_env, ty, use_cloned_def_id)
     }
 
     fn type_is_sized_modulo_regions(&self, param_env: ty::ParamEnv<'tcx>, ty: Ty<'tcx>) -> bool {
-        let lang_item = self.tcx.require_lang_item(LangItem::Sized, None);
+        let lang_item = self.tcx.require_lang_item(LangItem::Sized, DUMMY_SP);
         traits::type_known_to_meet_bound_modulo_regions(self, param_env, ty, lang_item)
     }
 
diff --git a/compiler/rustc_trait_selection/src/lib.rs b/compiler/rustc_trait_selection/src/lib.rs
index 67328defe36..e2b22f7bab7 100644
--- a/compiler/rustc_trait_selection/src/lib.rs
+++ b/compiler/rustc_trait_selection/src/lib.rs
@@ -19,14 +19,12 @@
 #![feature(assert_matches)]
 #![feature(associated_type_defaults)]
 #![feature(box_patterns)]
-#![feature(cfg_version)]
 #![feature(if_let_guard)]
 #![feature(iter_intersperse)]
 #![feature(iterator_try_reduce)]
 #![feature(never_type)]
 #![feature(rustdoc_internals)]
 #![feature(try_blocks)]
-#![feature(type_alias_impl_trait)]
 #![feature(unwrap_infallible)]
 #![feature(yeet_expr)]
 #![recursion_limit = "512"] // For rustdoc
diff --git a/compiler/rustc_trait_selection/src/opaque_types.rs b/compiler/rustc_trait_selection/src/opaque_types.rs
index 332204a0c5f..d5bde9192d5 100644
--- a/compiler/rustc_trait_selection/src/opaque_types.rs
+++ b/compiler/rustc_trait_selection/src/opaque_types.rs
@@ -81,7 +81,7 @@ pub fn check_opaque_type_parameter_valid<'tcx>(
     }
 
     for (i, arg) in opaque_type_key.iter_captured_args(tcx) {
-        let arg_is_param = match arg.unpack() {
+        let arg_is_param = match arg.kind() {
             GenericArgKind::Lifetime(lt) => match defining_scope_kind {
                 DefiningScopeKind::HirTypeck => continue,
                 DefiningScopeKind::MirBorrowck => {
diff --git a/compiler/rustc_trait_selection/src/solve/delegate.rs b/compiler/rustc_trait_selection/src/solve/delegate.rs
index 3601c2cba9b..69a0c0809b5 100644
--- a/compiler/rustc_trait_selection/src/solve/delegate.rs
+++ b/compiler/rustc_trait_selection/src/solve/delegate.rs
@@ -1,19 +1,22 @@
 use std::ops::Deref;
 
 use rustc_data_structures::fx::FxHashSet;
+use rustc_hir::LangItem;
 use rustc_hir::def_id::{CRATE_DEF_ID, DefId};
 use rustc_infer::infer::canonical::query_response::make_query_region_constraints;
 use rustc_infer::infer::canonical::{
-    Canonical, CanonicalExt as _, CanonicalQueryInput, CanonicalVarInfo, CanonicalVarValues,
+    Canonical, CanonicalExt as _, CanonicalQueryInput, CanonicalVarKind, CanonicalVarValues,
 };
-use rustc_infer::infer::{InferCtxt, RegionVariableOrigin, TyCtxtInferExt};
+use rustc_infer::infer::{InferCtxt, RegionVariableOrigin, SubregionOrigin, TyCtxtInferExt};
 use rustc_infer::traits::solve::Goal;
 use rustc_middle::traits::query::NoSolution;
 use rustc_middle::traits::solve::Certainty;
-use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable, TypeVisitableExt as _, TypingMode};
+use rustc_middle::ty::{
+    self, Ty, TyCtxt, TypeFlags, TypeFoldable, TypeVisitableExt as _, TypingMode,
+};
 use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span};
 
-use crate::traits::{EvaluateConstErr, specialization_graph};
+use crate::traits::{EvaluateConstErr, ObligationCause, specialization_graph};
 
 #[repr(transparent)]
 pub struct SolverDelegate<'tcx>(InferCtxt<'tcx>);
@@ -55,12 +58,94 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate<
         (SolverDelegate(infcx), value, vars)
     }
 
+    fn compute_goal_fast_path(
+        &self,
+        goal: Goal<'tcx, ty::Predicate<'tcx>>,
+        span: Span,
+    ) -> Option<Certainty> {
+        if let Some(trait_pred) = goal.predicate.as_trait_clause() {
+            if self.shallow_resolve(trait_pred.self_ty().skip_binder()).is_ty_var()
+                // We don't do this fast path when opaques are defined since we may
+                // eventually use opaques to incompletely guide inference via ty var
+                // self types.
+                // FIXME: Properly consider opaques here.
+                && self.inner.borrow_mut().opaque_types().is_empty()
+            {
+                return Some(Certainty::AMBIGUOUS);
+            }
+
+            if trait_pred.polarity() == ty::PredicatePolarity::Positive {
+                match self.0.tcx.as_lang_item(trait_pred.def_id()) {
+                    Some(LangItem::Sized)
+                        if self
+                            .resolve_vars_if_possible(trait_pred.self_ty().skip_binder())
+                            .is_trivially_sized(self.0.tcx) =>
+                    {
+                        return Some(Certainty::Yes);
+                    }
+                    Some(LangItem::Copy | LangItem::Clone) => {
+                        let self_ty =
+                            self.resolve_vars_if_possible(trait_pred.self_ty().skip_binder());
+                        // Unlike `Sized` traits, which always prefer the built-in impl,
+                        // `Copy`/`Clone` may be shadowed by a param-env candidate which
+                        // could force a lifetime error or guide inference. While that's
+                        // not generally desirable, it is observable, so for now let's
+                        // ignore this fast path for types that have regions or infer.
+                        if !self_ty
+                            .has_type_flags(TypeFlags::HAS_FREE_REGIONS | TypeFlags::HAS_INFER)
+                            && self_ty.is_trivially_pure_clone_copy()
+                        {
+                            return Some(Certainty::Yes);
+                        }
+                    }
+                    _ => {}
+                }
+            }
+        }
+
+        let pred = goal.predicate.kind();
+        match pred.no_bound_vars()? {
+            ty::PredicateKind::DynCompatible(def_id) if self.0.tcx.is_dyn_compatible(def_id) => {
+                Some(Certainty::Yes)
+            }
+            ty::PredicateKind::Clause(ty::ClauseKind::RegionOutlives(outlives)) => {
+                self.0.sub_regions(
+                    SubregionOrigin::RelateRegionParamBound(span, None),
+                    outlives.1,
+                    outlives.0,
+                );
+                Some(Certainty::Yes)
+            }
+            ty::PredicateKind::Clause(ty::ClauseKind::TypeOutlives(outlives)) => {
+                self.0.register_type_outlives_constraint(
+                    outlives.0,
+                    outlives.1,
+                    &ObligationCause::dummy_with_span(span),
+                );
+
+                Some(Certainty::Yes)
+            }
+            ty::PredicateKind::Subtype(ty::SubtypePredicate { a, b, .. })
+            | ty::PredicateKind::Coerce(ty::CoercePredicate { a, b }) => {
+                if self.shallow_resolve(a).is_ty_var() && self.shallow_resolve(b).is_ty_var() {
+                    // FIXME: We also need to register a subtype relation between these vars
+                    // when those are added, and if they aren't in the same sub root then
+                    // we should mark this goal as `has_changed`.
+                    Some(Certainty::AMBIGUOUS)
+                } else {
+                    None
+                }
+            }
+            _ => None,
+        }
+    }
+
     fn fresh_var_for_kind_with_span(
         &self,
         arg: ty::GenericArg<'tcx>,
         span: Span,
     ) -> ty::GenericArg<'tcx> {
-        match arg.unpack() {
+        match arg.kind() {
             ty::GenericArgKind::Lifetime(_) => {
                 self.next_region_var(RegionVariableOrigin::MiscVariable(span)).into()
             }
@@ -142,11 +227,11 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate<
 
     fn instantiate_canonical_var_with_infer(
         &self,
-        cv_info: CanonicalVarInfo<'tcx>,
+        kind: CanonicalVarKind<'tcx>,
         span: Span,
         universe_map: impl Fn(ty::UniverseIndex) -> ty::UniverseIndex,
     ) -> ty::GenericArg<'tcx> {
-        self.0.instantiate_canonical_var(span, cv_info, universe_map)
+        self.0.instantiate_canonical_var(span, kind, universe_map)
     }
 
     fn add_item_bounds_for_hidden_type(
diff --git a/compiler/rustc_trait_selection/src/solve/fulfill.rs b/compiler/rustc_trait_selection/src/solve/fulfill.rs
index 3e1cdac84df..ed99c678a4d 100644
--- a/compiler/rustc_trait_selection/src/solve/fulfill.rs
+++ b/compiler/rustc_trait_selection/src/solve/fulfill.rs
@@ -12,8 +12,12 @@ use rustc_infer::traits::{
 use rustc_middle::ty::{
     self, DelayedSet, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitor, TypingMode,
 };
-use rustc_next_trait_solver::solve::{GenerateProofTree, HasChanged, SolverDelegateEvalExt as _};
+use rustc_next_trait_solver::delegate::SolverDelegate as _;
+use rustc_next_trait_solver::solve::{
+    GenerateProofTree, GoalEvaluation, GoalStalledOn, HasChanged, SolverDelegateEvalExt as _,
+};
 use rustc_span::Span;
+use thin_vec::ThinVec;
 use tracing::instrument;
 
 use self::derive_errors::*;
@@ -24,6 +28,10 @@ use crate::traits::{FulfillmentError, ScrubbedTraitError};
 
 mod derive_errors;
 
+// FIXME: Do we need to use a `ThinVec` here?
+type PendingObligations<'tcx> =
+    ThinVec<(PredicateObligation<'tcx>, Option<GoalStalledOn<TyCtxt<'tcx>>>)>;
+
 /// A trait engine using the new trait solver.
 ///
 /// This is mostly identical to how `evaluate_all` works inside of the
@@ -53,13 +61,17 @@ struct ObligationStorage<'tcx> {
     /// We cannot eagerly return these as error so we instead store them here
     /// to avoid recomputing them each time `select_where_possible` is called.
     /// This also allows us to return the correct `FulfillmentError` for them.
-    overflowed: PredicateObligations<'tcx>,
-    pending: PredicateObligations<'tcx>,
+    overflowed: Vec<PredicateObligation<'tcx>>,
+    pending: PendingObligations<'tcx>,
 }
 
 impl<'tcx> ObligationStorage<'tcx> {
-    fn register(&mut self, obligation: PredicateObligation<'tcx>) {
-        self.pending.push(obligation);
+    fn register(
+        &mut self,
+        obligation: PredicateObligation<'tcx>,
+        stalled_on: Option<GoalStalledOn<TyCtxt<'tcx>>>,
+    ) {
+        self.pending.push((obligation, stalled_on));
     }
 
     fn has_pending_obligations(&self) -> bool {
@@ -67,7 +79,8 @@ impl<'tcx> ObligationStorage<'tcx> {
     }
 
     fn clone_pending(&self) -> PredicateObligations<'tcx> {
-        let mut obligations = self.pending.clone();
+        let mut obligations: PredicateObligations<'tcx> =
+            self.pending.iter().map(|(o, _)| o.clone()).collect();
         obligations.extend(self.overflowed.iter().cloned());
         obligations
     }
@@ -75,8 +88,9 @@ impl<'tcx> ObligationStorage<'tcx> {
     fn drain_pending(
         &mut self,
         cond: impl Fn(&PredicateObligation<'tcx>) -> bool,
-    ) -> PredicateObligations<'tcx> {
-        let (unstalled, pending) = mem::take(&mut self.pending).into_iter().partition(cond);
+    ) -> PendingObligations<'tcx> {
+        let (unstalled, pending) =
+            mem::take(&mut self.pending).into_iter().partition(|(o, _)| cond(o));
         self.pending = pending;
         unstalled
     }
@@ -89,13 +103,21 @@ impl<'tcx> ObligationStorage<'tcx> {
             // we were to do another step of `select_where_possible`, which goals would
             // change.
             // FIXME: <https://github.com/Gankra/thin-vec/pull/66> is merged, this can be removed.
-            self.overflowed.extend(ExtractIf::new(&mut self.pending, |o| {
-                let goal = o.as_goal();
-                let result = <&SolverDelegate<'tcx>>::from(infcx)
-                    .evaluate_root_goal(goal, GenerateProofTree::No, o.cause.span)
-                    .0;
-                matches!(result, Ok((HasChanged::Yes, _)))
-            }));
+            self.overflowed.extend(
+                ExtractIf::new(&mut self.pending, |(o, stalled_on)| {
+                    let goal = o.as_goal();
+                    let result = <&SolverDelegate<'tcx>>::from(infcx)
+                        .evaluate_root_goal(
+                            goal,
+                            GenerateProofTree::No,
+                            o.cause.span,
+                            stalled_on.take(),
+                        )
+                        .0;
+                    matches!(result, Ok(GoalEvaluation { has_changed: HasChanged::Yes, .. }))
+                })
+                .map(|(o, _)| o),
+            );
         })
     }
 }
@@ -118,11 +140,11 @@ impl<'tcx, E: 'tcx> FulfillmentCtxt<'tcx, E> {
         &self,
         infcx: &InferCtxt<'tcx>,
         obligation: &PredicateObligation<'tcx>,
-        result: &Result<(HasChanged, Certainty), NoSolution>,
+        result: &Result<GoalEvaluation<TyCtxt<'tcx>>, NoSolution>,
     ) {
         if let Some(inspector) = infcx.obligation_inspector.get() {
             let result = match result {
-                Ok((_, c)) => Ok(*c),
+                Ok(GoalEvaluation { certainty, .. }) => Ok(*certainty),
                 Err(NoSolution) => Err(NoSolution),
             };
             (inspector)(infcx, &obligation, result);
@@ -141,14 +163,14 @@ where
         obligation: PredicateObligation<'tcx>,
     ) {
         assert_eq!(self.usable_in_snapshot, infcx.num_open_snapshots());
-        self.obligations.register(obligation);
+        self.obligations.register(obligation, None);
     }
 
     fn collect_remaining_errors(&mut self, infcx: &InferCtxt<'tcx>) -> Vec<E> {
         self.obligations
             .pending
             .drain(..)
-            .map(|obligation| NextSolverError::Ambiguity(obligation))
+            .map(|(obligation, _)| NextSolverError::Ambiguity(obligation))
             .chain(
                 self.obligations
                     .overflowed
@@ -163,8 +185,8 @@ where
         assert_eq!(self.usable_in_snapshot, infcx.num_open_snapshots());
         let mut errors = Vec::new();
         loop {
-            let mut has_changed = false;
-            for mut obligation in self.obligations.drain_pending(|_| true) {
+            let mut any_changed = false;
+            for (mut obligation, stalled_on) in self.obligations.drain_pending(|_| true) {
                 if !infcx.tcx.recursion_limit().value_within_limit(obligation.recursion_depth) {
                     self.obligations.on_fulfillment_overflow(infcx);
                     // Only return true errors that we have accumulated while processing.
@@ -172,11 +194,29 @@ where
                 }
 
                 let goal = obligation.as_goal();
-                let result = <&SolverDelegate<'tcx>>::from(infcx)
-                    .evaluate_root_goal(goal, GenerateProofTree::No, obligation.cause.span)
+                let delegate = <&SolverDelegate<'tcx>>::from(infcx);
+                if let Some(certainty) =
+                    delegate.compute_goal_fast_path(goal, obligation.cause.span)
+                {
+                    match certainty {
+                        Certainty::Yes => {}
+                        Certainty::Maybe(_) => {
+                            self.obligations.register(obligation, None);
+                        }
+                    }
+                    continue;
+                }
+
+                let result = delegate
+                    .evaluate_root_goal(
+                        goal,
+                        GenerateProofTree::No,
+                        obligation.cause.span,
+                        stalled_on,
+                    )
                     .0;
                 self.inspect_evaluated_obligation(infcx, &obligation, &result);
-                let (changed, certainty) = match result {
+                let GoalEvaluation { certainty, has_changed, stalled_on } = match result {
                     Ok(result) => result,
                     Err(NoSolution) => {
                         errors.push(E::from_solver_error(
@@ -187,7 +227,7 @@ where
                     }
                 };
 
-                if changed == HasChanged::Yes {
+                if has_changed == HasChanged::Yes {
                     // We increment the recursion depth here to track the number of times
                     // this goal has resulted in inference progress. This doesn't precisely
                     // model the way that we track recursion depth in the old solver due
@@ -195,16 +235,16 @@ where
                     // approximation and should only result in fulfillment overflow in
                     // pathological cases.
                     obligation.recursion_depth += 1;
-                    has_changed = true;
+                    any_changed = true;
                 }
 
                 match certainty {
                     Certainty::Yes => {}
-                    Certainty::Maybe(_) => self.obligations.register(obligation),
+                    Certainty::Maybe(_) => self.obligations.register(obligation, stalled_on),
                 }
             }
 
-            if !has_changed {
+            if !any_changed {
                 break;
             }
         }
@@ -238,20 +278,24 @@ where
             return Default::default();
         }
 
-        self.obligations.drain_pending(|obl| {
-            infcx.probe(|_| {
-                infcx
-                    .visit_proof_tree(
-                        obl.as_goal(),
-                        &mut StalledOnCoroutines {
-                            stalled_generators,
-                            span: obl.cause.span,
-                            cache: Default::default(),
-                        },
-                    )
-                    .is_break()
+        self.obligations
+            .drain_pending(|obl| {
+                infcx.probe(|_| {
+                    infcx
+                        .visit_proof_tree(
+                            obl.as_goal(),
+                            &mut StalledOnCoroutines {
+                                stalled_generators,
+                                span: obl.cause.span,
+                                cache: Default::default(),
+                            },
+                        )
+                        .is_break()
+                })
             })
-        })
+            .into_iter()
+            .map(|(o, _)| o)
+            .collect()
     }
 }
 
diff --git a/compiler/rustc_trait_selection/src/solve/fulfill/derive_errors.rs b/compiler/rustc_trait_selection/src/solve/fulfill/derive_errors.rs
index f64cd5ffebe..36a8ae675c0 100644
--- a/compiler/rustc_trait_selection/src/solve/fulfill/derive_errors.rs
+++ b/compiler/rustc_trait_selection/src/solve/fulfill/derive_errors.rs
@@ -11,7 +11,9 @@ use rustc_middle::traits::query::NoSolution;
 use rustc_middle::ty::error::{ExpectedFound, TypeError};
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_middle::{bug, span_bug};
-use rustc_next_trait_solver::solve::{GenerateProofTree, SolverDelegateEvalExt as _};
+use rustc_next_trait_solver::solve::{
+    GenerateProofTree, GoalEvaluation, SolverDelegateEvalExt as _,
+};
 use tracing::{instrument, trace};
 
 use crate::solve::delegate::SolverDelegate;
@@ -93,19 +95,21 @@ pub(super) fn fulfillment_error_for_stalled<'tcx>(
                 root_obligation.as_goal(),
                 GenerateProofTree::No,
                 root_obligation.cause.span,
+                None,
             )
             .0
         {
-            Ok((_, Certainty::Maybe(MaybeCause::Ambiguity))) => {
+            Ok(GoalEvaluation { certainty: Certainty::Maybe(MaybeCause::Ambiguity), .. }) => {
                 (FulfillmentErrorCode::Ambiguity { overflow: None }, true)
             }
-            Ok((
-                _,
-                Certainty::Maybe(MaybeCause::Overflow {
-                    suggest_increasing_limit,
-                    keep_constraints: _,
-                }),
-            )) => (
+            Ok(GoalEvaluation {
+                certainty:
+                    Certainty::Maybe(MaybeCause::Overflow {
+                        suggest_increasing_limit,
+                        keep_constraints: _,
+                    }),
+                ..
+            }) => (
                 FulfillmentErrorCode::Ambiguity { overflow: Some(suggest_increasing_limit) },
                 // Don't look into overflows because we treat overflows weirdly anyways.
                 // We discard the inference constraints from overflowing goals, so
@@ -115,14 +119,16 @@ pub(super) fn fulfillment_error_for_stalled<'tcx>(
                 // FIXME: We should probably just look into overflows here.
                 false,
             ),
-            Ok((_, Certainty::Yes)) => {
-                bug!(
+            Ok(GoalEvaluation { certainty: Certainty::Yes, .. }) => {
+                span_bug!(
+                    root_obligation.cause.span,
                     "did not expect successful goal when collecting ambiguity errors for `{:?}`",
                     infcx.resolve_vars_if_possible(root_obligation.predicate),
                 )
             }
             Err(_) => {
-                bug!(
+                span_bug!(
+                    root_obligation.cause.span,
                     "did not expect selection error when collecting ambiguity errors for `{:?}`",
                     infcx.resolve_vars_if_possible(root_obligation.predicate),
                 )
diff --git a/compiler/rustc_trait_selection/src/solve/inspect/analyse.rs b/compiler/rustc_trait_selection/src/solve/inspect/analyse.rs
index 9795655e842..d5d318ee490 100644
--- a/compiler/rustc_trait_selection/src/solve/inspect/analyse.rs
+++ b/compiler/rustc_trait_selection/src/solve/inspect/analyse.rs
@@ -15,9 +15,9 @@ use rustc_infer::infer::{DefineOpaqueTypes, InferCtxt, InferOk};
 use rustc_macros::extension;
 use rustc_middle::traits::ObligationCause;
 use rustc_middle::traits::solve::{Certainty, Goal, GoalSource, NoSolution, QueryResult};
-use rustc_middle::ty::{TyCtxt, TypeFoldable, VisitorResult, try_visit};
+use rustc_middle::ty::{TyCtxt, VisitorResult, try_visit};
 use rustc_middle::{bug, ty};
-use rustc_next_trait_solver::resolve::EagerResolver;
+use rustc_next_trait_solver::resolve::eager_resolve_vars;
 use rustc_next_trait_solver::solve::inspect::{self, instantiate_canonical_state};
 use rustc_next_trait_solver::solve::{GenerateProofTree, MaybeCause, SolverDelegateEvalExt as _};
 use rustc_span::{DUMMY_SP, Span};
@@ -133,45 +133,25 @@ impl<'a, 'tcx> InspectCandidate<'a, 'tcx> {
     /// Instantiate the nested goals for the candidate without rolling back their
     /// inference constraints. This function modifies the state of the `infcx`.
     ///
-    /// See [`Self::instantiate_nested_goals_and_opt_impl_args`] if you need the impl args too.
-    pub fn instantiate_nested_goals(&self, span: Span) -> Vec<InspectGoal<'a, 'tcx>> {
-        self.instantiate_nested_goals_and_opt_impl_args(span).0
-    }
-
-    /// Instantiate the nested goals for the candidate without rolling back their
-    /// inference constraints, and optionally the args of an impl if this candidate
-    /// came from a `CandidateSource::Impl`. This function modifies the state of the
-    /// `infcx`.
+    /// See [`Self::instantiate_impl_args`] if you need the impl args too.
     #[instrument(
         level = "debug",
         skip_all,
         fields(goal = ?self.goal.goal, steps = ?self.steps)
     )]
-    pub fn instantiate_nested_goals_and_opt_impl_args(
-        &self,
-        span: Span,
-    ) -> (Vec<InspectGoal<'a, 'tcx>>, Option<ty::GenericArgsRef<'tcx>>) {
+    pub fn instantiate_nested_goals(&self, span: Span) -> Vec<InspectGoal<'a, 'tcx>> {
         let infcx = self.goal.infcx;
         let param_env = self.goal.goal.param_env;
         let mut orig_values = self.goal.orig_values.to_vec();
 
         let mut instantiated_goals = vec![];
-        let mut opt_impl_args = None;
         for step in &self.steps {
             match **step {
                 inspect::ProbeStep::AddGoal(source, goal) => instantiated_goals.push((
                     source,
                     instantiate_canonical_state(infcx, span, param_env, &mut orig_values, goal),
                 )),
-                inspect::ProbeStep::RecordImplArgs { impl_args } => {
-                    opt_impl_args = Some(instantiate_canonical_state(
-                        infcx,
-                        span,
-                        param_env,
-                        &mut orig_values,
-                        impl_args,
-                    ));
-                }
+                inspect::ProbeStep::RecordImplArgs { .. } => {}
                 inspect::ProbeStep::MakeCanonicalResponse { .. }
                 | inspect::ProbeStep::NestedProbe(_) => unreachable!(),
             }
@@ -187,15 +167,59 @@ impl<'a, 'tcx> InspectCandidate<'a, 'tcx> {
             let _ = term_hack.constrain(infcx, span, param_env);
         }
 
-        let opt_impl_args =
-            opt_impl_args.map(|impl_args| impl_args.fold_with(&mut EagerResolver::new(infcx)));
-
-        let goals = instantiated_goals
+        instantiated_goals
             .into_iter()
             .map(|(source, goal)| self.instantiate_proof_tree_for_nested_goal(source, goal, span))
-            .collect();
+            .collect()
+    }
+
+    /// Instantiate the args of an impl if this candidate came from a
+    /// `CandidateSource::Impl`. This function modifies the state of the
+    /// `infcx`.
+    #[instrument(
+        level = "debug",
+        skip_all,
+        fields(goal = ?self.goal.goal, steps = ?self.steps)
+    )]
+    pub fn instantiate_impl_args(&self, span: Span) -> ty::GenericArgsRef<'tcx> {
+        let infcx = self.goal.infcx;
+        let param_env = self.goal.goal.param_env;
+        let mut orig_values = self.goal.orig_values.to_vec();
+
+        for step in &self.steps {
+            match **step {
+                inspect::ProbeStep::RecordImplArgs { impl_args } => {
+                    let impl_args = instantiate_canonical_state(
+                        infcx,
+                        span,
+                        param_env,
+                        &mut orig_values,
+                        impl_args,
+                    );
+
+                    let () = instantiate_canonical_state(
+                        infcx,
+                        span,
+                        param_env,
+                        &mut orig_values,
+                        self.final_state,
+                    );
 
-        (goals, opt_impl_args)
+                    // No reason we couldn't support this, but we don't need to for select.
+                    assert!(
+                        self.goal.normalizes_to_term_hack.is_none(),
+                        "cannot use `instantiate_impl_args` with a `NormalizesTo` goal"
+                    );
+
+                    return eager_resolve_vars(infcx, impl_args);
+                }
+                inspect::ProbeStep::AddGoal(..) => {}
+                inspect::ProbeStep::MakeCanonicalResponse { .. }
+                | inspect::ProbeStep::NestedProbe(_) => unreachable!(),
+            }
+        }
+
+        bug!("expected impl args probe step for `instantiate_impl_args`");
     }
 
     pub fn instantiate_proof_tree_for_nested_goal(
@@ -207,10 +231,7 @@ impl<'a, 'tcx> InspectCandidate<'a, 'tcx> {
         let infcx = self.goal.infcx;
         match goal.predicate.kind().no_bound_vars() {
             Some(ty::PredicateKind::NormalizesTo(ty::NormalizesTo { alias, term })) => {
-                let unconstrained_term = match term.unpack() {
-                    ty::TermKind::Ty(_) => infcx.next_ty_var(span).into(),
-                    ty::TermKind::Const(_) => infcx.next_const_var(span).into(),
-                };
+                let unconstrained_term = infcx.next_term_var_of_kind(term, span);
                 let goal =
                     goal.with(infcx.tcx, ty::NormalizesTo { alias, term: unconstrained_term });
                 // We have to use a `probe` here as evaluating a `NormalizesTo` can constrain the
@@ -219,8 +240,8 @@ impl<'a, 'tcx> InspectCandidate<'a, 'tcx> {
                 // building their proof tree, the expected term was unconstrained, but when
                 // instantiating the candidate it is already constrained to the result of another
                 // candidate.
-                let proof_tree =
-                    infcx.probe(|_| infcx.evaluate_root_goal_raw(goal, GenerateProofTree::Yes).1);
+                let proof_tree = infcx
+                    .probe(|_| infcx.evaluate_root_goal_raw(goal, GenerateProofTree::Yes, None).1);
                 InspectGoal::new(
                     infcx,
                     self.goal.depth + 1,
@@ -236,7 +257,7 @@ impl<'a, 'tcx> InspectCandidate<'a, 'tcx> {
                 // constraints, we get an ICE if we already applied the constraints
                 // from the chosen candidate.
                 let proof_tree = infcx
-                    .probe(|_| infcx.evaluate_root_goal(goal, GenerateProofTree::Yes, span).1)
+                    .probe(|_| infcx.evaluate_root_goal(goal, GenerateProofTree::Yes, span, None).1)
                     .unwrap();
                 InspectGoal::new(infcx, self.goal.depth + 1, proof_tree, None, source)
             }
@@ -392,7 +413,7 @@ impl<'a, 'tcx> InspectGoal<'a, 'tcx> {
             infcx,
             depth,
             orig_values,
-            goal: uncanonicalized_goal.fold_with(&mut EagerResolver::new(infcx)),
+            goal: eager_resolve_vars(infcx, uncanonicalized_goal),
             result,
             evaluation_kind: evaluation.kind,
             normalizes_to_term_hack,
@@ -442,6 +463,7 @@ impl<'tcx> InferCtxt<'tcx> {
             goal,
             GenerateProofTree::Yes,
             visitor.span(),
+            None,
         );
         let proof_tree = proof_tree.unwrap();
         visitor.visit_goal(&InspectGoal::new(self, depth, proof_tree, None, GoalSource::Misc))
diff --git a/compiler/rustc_trait_selection/src/solve/normalize.rs b/compiler/rustc_trait_selection/src/solve/normalize.rs
index d903f94b489..8f44c26b70d 100644
--- a/compiler/rustc_trait_selection/src/solve/normalize.rs
+++ b/compiler/rustc_trait_selection/src/solve/normalize.rs
@@ -1,4 +1,3 @@
-use std::assert_matches::assert_matches;
 use std::fmt::Debug;
 
 use rustc_data_structures::stack::ensure_sufficient_stack;
@@ -16,7 +15,6 @@ use tracing::instrument;
 use super::{FulfillmentCtxt, NextSolverError};
 use crate::error_reporting::InferCtxtErrorExt;
 use crate::error_reporting::traits::OverflowCause;
-use crate::traits::query::evaluate_obligation::InferCtxtExt;
 use crate::traits::{BoundVarReplacer, PlaceholderReplacer, ScrubbedTraitError};
 
 /// Deeply normalize all aliases in `value`. This does not handle inference and expects
@@ -97,19 +95,18 @@ impl<'tcx, E> NormalizationFolder<'_, 'tcx, E>
 where
     E: FromSolverError<'tcx, NextSolverError<'tcx>>,
 {
-    fn normalize_alias_ty(&mut self, alias_ty: Ty<'tcx>) -> Result<Ty<'tcx>, Vec<E>> {
-        assert_matches!(alias_ty.kind(), ty::Alias(..));
-
+    fn normalize_alias_term(
+        &mut self,
+        alias_term: ty::Term<'tcx>,
+    ) -> Result<ty::Term<'tcx>, Vec<E>> {
         let infcx = self.at.infcx;
         let tcx = infcx.tcx;
         let recursion_limit = tcx.recursion_limit();
         if !recursion_limit.value_within_limit(self.depth) {
-            let ty::Alias(_, data) = *alias_ty.kind() else {
-                unreachable!();
-            };
+            let term = alias_term.to_alias_term().unwrap();
 
             self.at.infcx.err_ctxt().report_overflow_error(
-                OverflowCause::DeeplyNormalize(data.into()),
+                OverflowCause::DeeplyNormalize(term),
                 self.at.cause.span,
                 true,
                 |_| {},
@@ -118,14 +115,14 @@ where
 
         self.depth += 1;
 
-        let new_infer_ty = infcx.next_ty_var(self.at.cause.span);
+        let infer_term = infcx.next_term_var_of_kind(alias_term, self.at.cause.span);
         let obligation = Obligation::new(
             tcx,
             self.at.cause.clone(),
             self.at.param_env,
             ty::PredicateKind::AliasRelate(
-                alias_ty.into(),
-                new_infer_ty.into(),
+                alias_term.into(),
+                infer_term.into(),
                 ty::AliasRelationDirection::Equate,
             ),
         );
@@ -135,50 +132,13 @@ where
 
         // Alias is guaranteed to be fully structurally resolved,
         // so we can super fold here.
-        let ty = infcx.resolve_vars_if_possible(new_infer_ty);
-        let result = ty.try_super_fold_with(self)?;
-        self.depth -= 1;
-        Ok(result)
-    }
-
-    fn normalize_unevaluated_const(
-        &mut self,
-        uv: ty::UnevaluatedConst<'tcx>,
-    ) -> Result<ty::Const<'tcx>, Vec<E>> {
-        let infcx = self.at.infcx;
-        let tcx = infcx.tcx;
-        let recursion_limit = tcx.recursion_limit();
-        if !recursion_limit.value_within_limit(self.depth) {
-            self.at.infcx.err_ctxt().report_overflow_error(
-                OverflowCause::DeeplyNormalize(uv.into()),
-                self.at.cause.span,
-                true,
-                |_| {},
-            );
-        }
-
-        self.depth += 1;
-
-        let new_infer_ct = infcx.next_const_var(self.at.cause.span);
-        let obligation = Obligation::new(
-            tcx,
-            self.at.cause.clone(),
-            self.at.param_env,
-            ty::NormalizesTo { alias: uv.into(), term: new_infer_ct.into() },
-        );
-
-        let result = if infcx.predicate_may_hold(&obligation) {
-            self.fulfill_cx.register_predicate_obligation(infcx, obligation);
-            let errors = self.fulfill_cx.select_where_possible(infcx);
-            if !errors.is_empty() {
-                return Err(errors);
-            }
-            let ct = infcx.resolve_vars_if_possible(new_infer_ct);
-            ct.try_fold_with(self)?
-        } else {
-            ty::Const::new_unevaluated(tcx, uv).try_super_fold_with(self)?
+        let term = infcx.resolve_vars_if_possible(infer_term);
+        // super-folding the `term` will directly fold the `Ty` or `Const` so
+        // we have to match on the term and super-fold them manually.
+        let result = match term.kind() {
+            ty::TermKind::Ty(ty) => ty.try_super_fold_with(self)?.into(),
+            ty::TermKind::Const(ct) => ct.try_super_fold_with(self)?.into(),
         };
-
         self.depth -= 1;
         Ok(result)
     }
@@ -238,7 +198,8 @@ where
         if ty.has_escaping_bound_vars() {
             let (ty, mapped_regions, mapped_types, mapped_consts) =
                 BoundVarReplacer::replace_bound_vars(infcx, &mut self.universes, ty);
-            let result = ensure_sufficient_stack(|| self.normalize_alias_ty(ty))?;
+            let result =
+                ensure_sufficient_stack(|| self.normalize_alias_term(ty.into()))?.expect_type();
             Ok(PlaceholderReplacer::replace_placeholders(
                 infcx,
                 mapped_regions,
@@ -248,7 +209,7 @@ where
                 result,
             ))
         } else {
-            ensure_sufficient_stack(|| self.normalize_alias_ty(ty))
+            Ok(ensure_sufficient_stack(|| self.normalize_alias_term(ty.into()))?.expect_type())
         }
     }
 
@@ -260,15 +221,13 @@ where
             return Ok(ct);
         }
 
-        let uv = match ct.kind() {
-            ty::ConstKind::Unevaluated(ct) => ct,
-            _ => return ct.try_super_fold_with(self),
-        };
+        let ty::ConstKind::Unevaluated(..) = ct.kind() else { return ct.try_super_fold_with(self) };
 
-        if uv.has_escaping_bound_vars() {
-            let (uv, mapped_regions, mapped_types, mapped_consts) =
-                BoundVarReplacer::replace_bound_vars(infcx, &mut self.universes, uv);
-            let result = ensure_sufficient_stack(|| self.normalize_unevaluated_const(uv))?;
+        if ct.has_escaping_bound_vars() {
+            let (ct, mapped_regions, mapped_types, mapped_consts) =
+                BoundVarReplacer::replace_bound_vars(infcx, &mut self.universes, ct);
+            let result =
+                ensure_sufficient_stack(|| self.normalize_alias_term(ct.into()))?.expect_const();
             Ok(PlaceholderReplacer::replace_placeholders(
                 infcx,
                 mapped_regions,
@@ -278,7 +237,7 @@ where
                 result,
             ))
         } else {
-            ensure_sufficient_stack(|| self.normalize_unevaluated_const(uv))
+            Ok(ensure_sufficient_stack(|| self.normalize_alias_term(ct.into()))?.expect_const())
         }
     }
 }
diff --git a/compiler/rustc_trait_selection/src/solve/select.rs b/compiler/rustc_trait_selection/src/solve/select.rs
index 4fdaf740287..fb1adc2fd2a 100644
--- a/compiler/rustc_trait_selection/src/solve/select.rs
+++ b/compiler/rustc_trait_selection/src/solve/select.rs
@@ -5,19 +5,21 @@ use rustc_infer::traits::solve::inspect::ProbeKind;
 use rustc_infer::traits::solve::{CandidateSource, Certainty, Goal};
 use rustc_infer::traits::{
     BuiltinImplSource, ImplSource, ImplSourceUserDefinedData, Obligation, ObligationCause,
-    PolyTraitObligation, Selection, SelectionError, SelectionResult,
+    Selection, SelectionError, SelectionResult, TraitObligation,
 };
 use rustc_macros::extension;
 use rustc_middle::{bug, span_bug};
 use rustc_span::Span;
+use thin_vec::thin_vec;
 
 use crate::solve::inspect::{self, ProofTreeInferCtxtExt};
 
 #[extension(pub trait InferCtxtSelectExt<'tcx>)]
 impl<'tcx> InferCtxt<'tcx> {
+    /// Do not use this directly. This is called from [`crate::traits::SelectionContext::select`].
     fn select_in_new_trait_solver(
         &self,
-        obligation: &PolyTraitObligation<'tcx>,
+        obligation: &TraitObligation<'tcx>,
     ) -> SelectionResult<'tcx, Selection<'tcx>> {
         assert!(self.next_trait_solver());
 
@@ -145,18 +147,21 @@ fn to_selection<'tcx>(
         return None;
     }
 
-    let (nested, impl_args) = cand.instantiate_nested_goals_and_opt_impl_args(span);
-    let nested = nested
-        .into_iter()
-        .map(|nested| {
-            Obligation::new(
-                nested.infcx().tcx,
-                ObligationCause::dummy_with_span(span),
-                nested.goal().param_env,
-                nested.goal().predicate,
-            )
-        })
-        .collect();
+    let nested = match cand.result().expect("expected positive result") {
+        Certainty::Yes => thin_vec![],
+        Certainty::Maybe(_) => cand
+            .instantiate_nested_goals(span)
+            .into_iter()
+            .map(|nested| {
+                Obligation::new(
+                    nested.infcx().tcx,
+                    ObligationCause::dummy_with_span(span),
+                    nested.goal().param_env,
+                    nested.goal().predicate,
+                )
+            })
+            .collect(),
+    };
 
     Some(match cand.kind() {
         ProbeKind::TraitCandidate { source, result: _ } => match source {
@@ -165,7 +170,7 @@ fn to_selection<'tcx>(
                 // For impl candidates, we do the rematch manually to compute the args.
                 ImplSource::UserDefined(ImplSourceUserDefinedData {
                     impl_def_id,
-                    args: impl_args.expect("expected recorded impl args for impl candidate"),
+                    args: cand.instantiate_impl_args(span),
                     nested,
                 })
             }
diff --git a/compiler/rustc_trait_selection/src/traits/auto_trait.rs b/compiler/rustc_trait_selection/src/traits/auto_trait.rs
index 02521c9453d..3ae908ec16b 100644
--- a/compiler/rustc_trait_selection/src/traits/auto_trait.rs
+++ b/compiler/rustc_trait_selection/src/traits/auto_trait.rs
@@ -726,7 +726,9 @@ impl<'tcx> AutoTraitFinder<'tcx> {
                 }
                 ty::PredicateKind::Clause(ty::ClauseKind::RegionOutlives(binder)) => {
                     let binder = bound_predicate.rebind(binder);
-                    selcx.infcx.region_outlives_predicate(&dummy_cause, binder)
+                    selcx.infcx.enter_forall(binder, |pred| {
+                        selcx.infcx.register_region_outlives_constraint(pred, &dummy_cause);
+                    });
                 }
                 ty::PredicateKind::Clause(ty::ClauseKind::TypeOutlives(binder)) => {
                     let binder = bound_predicate.rebind(binder);
@@ -735,14 +737,14 @@ impl<'tcx> AutoTraitFinder<'tcx> {
                         binder.map_bound_ref(|pred| pred.0).no_bound_vars(),
                     ) {
                         (None, Some(t_a)) => {
-                            selcx.infcx.register_region_obligation_with_cause(
+                            selcx.infcx.register_type_outlives_constraint(
                                 t_a,
                                 selcx.infcx.tcx.lifetimes.re_static,
                                 &dummy_cause,
                             );
                         }
                         (Some(ty::OutlivesPredicate(t_a, r_b)), _) => {
-                            selcx.infcx.register_region_obligation_with_cause(
+                            selcx.infcx.register_type_outlives_constraint(
                                 t_a,
                                 r_b,
                                 &dummy_cause,
diff --git a/compiler/rustc_trait_selection/src/traits/effects.rs b/compiler/rustc_trait_selection/src/traits/effects.rs
index cc5861b5a1f..e77d9e32cb9 100644
--- a/compiler/rustc_trait_selection/src/traits/effects.rs
+++ b/compiler/rustc_trait_selection/src/traits/effects.rs
@@ -248,7 +248,7 @@ fn evaluate_host_effect_for_destruct_goal<'tcx>(
     obligation: &HostEffectObligation<'tcx>,
 ) -> Result<ThinVec<PredicateObligation<'tcx>>, EvaluationFailure> {
     let tcx = selcx.tcx();
-    let destruct_def_id = tcx.require_lang_item(LangItem::Destruct, None);
+    let destruct_def_id = tcx.require_lang_item(LangItem::Destruct, obligation.cause.span);
     let self_ty = obligation.predicate.self_ty();
 
     let const_conditions = match *self_ty.kind() {
@@ -267,7 +267,7 @@ fn evaluate_host_effect_for_destruct_goal<'tcx>(
                 Some(hir::Constness::NotConst) => return Err(EvaluationFailure::NoSolution),
                 // `Drop` impl exists, and it's const. Require `Ty: ~const Drop` to hold.
                 Some(hir::Constness::Const) => {
-                    let drop_def_id = tcx.require_lang_item(LangItem::Drop, None);
+                    let drop_def_id = tcx.require_lang_item(LangItem::Drop, obligation.cause.span);
                     let drop_trait_ref = ty::TraitRef::new(tcx, drop_def_id, [self_ty]);
                     const_conditions.push(drop_trait_ref);
                 }
diff --git a/compiler/rustc_trait_selection/src/traits/engine.rs b/compiler/rustc_trait_selection/src/traits/engine.rs
index 8d6e6b4a651..18f28d72f6f 100644
--- a/compiler/rustc_trait_selection/src/traits/engine.rs
+++ b/compiler/rustc_trait_selection/src/traits/engine.rs
@@ -188,6 +188,20 @@ where
             .map(|infer_ok| self.register_infer_ok_obligations(infer_ok))
     }
 
+    /// Computes the least-upper-bound, or mutual supertype, of two values.
+    pub fn lub<T: ToTrace<'tcx>>(
+        &self,
+        cause: &ObligationCause<'tcx>,
+        param_env: ty::ParamEnv<'tcx>,
+        expected: T,
+        actual: T,
+    ) -> Result<T, TypeError<'tcx>> {
+        self.infcx
+            .at(cause, param_env)
+            .lub(expected, actual)
+            .map(|infer_ok| self.register_infer_ok_obligations(infer_ok))
+    }
+
     #[must_use]
     pub fn select_where_possible(&self) -> Vec<E> {
         self.engine.borrow_mut().select_where_possible(self.infcx)
diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs
index 34c3c905bd9..951dfb879ae 100644
--- a/compiler/rustc_trait_selection/src/traits/fulfill.rs
+++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs
@@ -428,7 +428,7 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
 
                 ty::PredicateKind::Clause(ty::ClauseKind::RegionOutlives(data)) => {
                     if infcx.considering_regions {
-                        infcx.region_outlives_predicate(&obligation.cause, Binder::dummy(data));
+                        infcx.register_region_outlives_constraint(data, &obligation.cause);
                     }
 
                     ProcessResult::Changed(Default::default())
@@ -439,7 +439,7 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
                     r_b,
                 ))) => {
                     if infcx.considering_regions {
-                        infcx.register_region_obligation_with_cause(t_a, r_b, &obligation.cause);
+                        infcx.register_type_outlives_constraint(t_a, r_b, &obligation.cause);
                     }
                     ProcessResult::Changed(Default::default())
                 }
diff --git a/compiler/rustc_trait_selection/src/traits/misc.rs b/compiler/rustc_trait_selection/src/traits/misc.rs
index a4b6f330b9d..393f458bea2 100644
--- a/compiler/rustc_trait_selection/src/traits/misc.rs
+++ b/compiler/rustc_trait_selection/src/traits/misc.rs
@@ -157,7 +157,7 @@ pub fn type_allowed_to_implement_const_param_ty<'tcx>(
             parent_cause.clone(),
             param_env,
             inner_ty,
-            tcx.require_lang_item(lang_item, Some(parent_cause.span)),
+            tcx.require_lang_item(lang_item, parent_cause.span),
         );
 
         let errors = ocx.select_all_or_error();
@@ -193,7 +193,7 @@ pub fn all_fields_implement_trait<'tcx>(
     parent_cause: ObligationCause<'tcx>,
     lang_item: LangItem,
 ) -> Result<(), Vec<(&'tcx ty::FieldDef, Ty<'tcx>, InfringingFieldsReason<'tcx>)>> {
-    let trait_def_id = tcx.require_lang_item(lang_item, Some(parent_cause.span));
+    let trait_def_id = tcx.require_lang_item(lang_item, parent_cause.span);
 
     let mut infringing = Vec::new();
     for variant in adt.variants() {
diff --git a/compiler/rustc_trait_selection/src/traits/mod.rs b/compiler/rustc_trait_selection/src/traits/mod.rs
index 31b075db04b..999ef97683c 100644
--- a/compiler/rustc_trait_selection/src/traits/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/mod.rs
@@ -221,7 +221,7 @@ fn pred_known_to_hold_modulo_regions<'tcx>(
 
     if result.must_apply_modulo_regions() {
         true
-    } else if result.may_apply() {
+    } else if result.may_apply() && !infcx.next_trait_solver() {
         // Sometimes obligations are ambiguous because the recursive evaluator
         // is not smart enough, so we fall back to fulfillment when we're not certain
         // that an obligation holds or not. Even still, we must make sure that
@@ -340,7 +340,7 @@ pub fn normalize_param_env_or_error<'tcx>(
     let mut predicates: Vec<_> = util::elaborate(
         tcx,
         unnormalized_env.caller_bounds().into_iter().map(|predicate| {
-            if tcx.features().generic_const_exprs() {
+            if tcx.features().generic_const_exprs() || tcx.next_trait_solver_globally() {
                 return predicate;
             }
 
@@ -405,8 +405,6 @@ pub fn normalize_param_env_or_error<'tcx>(
             // compatibility. Eventually when lazy norm is implemented this can just be removed.
             // We do not normalize types here as there is no backwards compatibility requirement
             // for us to do so.
-            //
-            // FIXME(-Znext-solver): remove this hack since we have deferred projection equality
             predicate.fold_with(&mut ConstNormalizer(tcx))
         }),
     )
@@ -542,10 +540,13 @@ pub fn try_evaluate_const<'tcx>(
         | ty::ConstKind::Placeholder(_)
         | ty::ConstKind::Expr(_) => Err(EvaluateConstErr::HasGenericsOrInfers),
         ty::ConstKind::Unevaluated(uv) => {
+            let opt_anon_const_kind =
+                (tcx.def_kind(uv.def) == DefKind::AnonConst).then(|| tcx.anon_const_kind(uv.def));
+
             // Postpone evaluation of constants that depend on generic parameters or
             // inference variables.
             //
-            // We use `TypingMode::PostAnalysis`  here which is not *technically* correct
+            // We use `TypingMode::PostAnalysis` here which is not *technically* correct
             // to be revealing opaque types here as borrowcheck has not run yet. However,
             // CTFE itself uses `TypingMode::PostAnalysis` unconditionally even during
             // typeck and not doing so has a lot of (undesirable) fallout (#101478, #119821).
@@ -553,65 +554,95 @@ pub fn try_evaluate_const<'tcx>(
             //
             // FIXME: `const_eval_resolve_for_typeck` should probably just modify the env itself
             // instead of having this logic here
-            let (args, typing_env) = if tcx.features().generic_const_exprs()
-                && uv.has_non_region_infer()
-            {
-                // `feature(generic_const_exprs)` causes anon consts to inherit all parent generics. This can cause
-                // inference variables and generic parameters to show up in `ty::Const` even though the anon const
-                // does not actually make use of them. We handle this case specially and attempt to evaluate anyway.
-                match tcx.thir_abstract_const(uv.def) {
-                    Ok(Some(ct)) => {
-                        let ct = tcx.expand_abstract_consts(ct.instantiate(tcx, uv.args));
-                        if let Err(e) = ct.error_reported() {
-                            return Err(EvaluateConstErr::EvaluationFailure(e));
-                        } else if ct.has_non_region_infer() || ct.has_non_region_param() {
-                            // If the anon const *does* actually use generic parameters or inference variables from
-                            // the generic arguments provided for it, then we should *not* attempt to evaluate it.
-                            return Err(EvaluateConstErr::HasGenericsOrInfers);
-                        } else {
-                            let args = replace_param_and_infer_args_with_placeholder(tcx, uv.args);
-                            let typing_env = infcx
-                                .typing_env(tcx.erase_regions(param_env))
-                                .with_post_analysis_normalized(tcx);
-                            (args, typing_env)
+            let (args, typing_env) = match opt_anon_const_kind {
+                // We handle `generic_const_exprs` separately as reasonable ways of handling constants in the type system
+                // completely fall apart under `generic_const_exprs` and makes this whole function Really hard to reason
+                // about if you have to consider gce whatsoever.
+                Some(ty::AnonConstKind::GCE) => {
+                    if uv.has_non_region_infer() || uv.has_non_region_param() {
+                        // `feature(generic_const_exprs)` causes anon consts to inherit all parent generics. This can cause
+                        // inference variables and generic parameters to show up in `ty::Const` even though the anon const
+                        // does not actually make use of them. We handle this case specially and attempt to evaluate anyway.
+                        match tcx.thir_abstract_const(uv.def) {
+                            Ok(Some(ct)) => {
+                                let ct = tcx.expand_abstract_consts(ct.instantiate(tcx, uv.args));
+                                if let Err(e) = ct.error_reported() {
+                                    return Err(EvaluateConstErr::EvaluationFailure(e));
+                                } else if ct.has_non_region_infer() || ct.has_non_region_param() {
+                                    // If the anon const *does* actually use generic parameters or inference variables from
+                                    // the generic arguments provided for it, then we should *not* attempt to evaluate it.
+                                    return Err(EvaluateConstErr::HasGenericsOrInfers);
+                                } else {
+                                    let args =
+                                        replace_param_and_infer_args_with_placeholder(tcx, uv.args);
+                                    let typing_env = infcx
+                                        .typing_env(tcx.erase_regions(param_env))
+                                        .with_post_analysis_normalized(tcx);
+                                    (args, typing_env)
+                                }
+                            }
+                            Err(_) | Ok(None) => {
+                                let args = GenericArgs::identity_for_item(tcx, uv.def);
+                                let typing_env = ty::TypingEnv::post_analysis(tcx, uv.def);
+                                (args, typing_env)
+                            }
                         }
+                    } else {
+                        let typing_env = infcx
+                            .typing_env(tcx.erase_regions(param_env))
+                            .with_post_analysis_normalized(tcx);
+                        (uv.args, typing_env)
                     }
-                    Err(_) | Ok(None) => {
-                        let args = GenericArgs::identity_for_item(tcx, uv.def);
-                        let typing_env = ty::TypingEnv::post_analysis(tcx, uv.def);
-                        (args, typing_env)
+                }
+                Some(ty::AnonConstKind::RepeatExprCount) => {
+                    if uv.has_non_region_infer() {
+                        // Diagnostics will sometimes replace the identity args of anon consts in
+                        // array repeat expr counts with inference variables so we have to handle this
+                        // even though it is not something we should ever actually encounter.
+                        //
+                        // Array repeat expr counts are allowed to syntactically use generic parameters
+                        // but must not actually depend on them in order to evalaute successfully. This means
+                        // that it is actually fine to evalaute them in their own environment rather than with
+                        // the actually provided generic arguments.
+                        tcx.dcx().delayed_bug("AnonConst with infer args but no error reported");
                     }
+
+                    // The generic args of repeat expr counts under `min_const_generics` are not supposed to
+                    // affect evaluation of the constant as this would make it a "truly" generic const arg.
+                    // To prevent this we discard all the generic arguments and evalaute with identity args
+                    // and in its own environment instead of the current environment we are normalizing in.
+                    let args = GenericArgs::identity_for_item(tcx, uv.def);
+                    let typing_env = ty::TypingEnv::post_analysis(tcx, uv.def);
+
+                    (args, typing_env)
+                }
+                _ => {
+                    // We are only dealing with "truly" generic/uninferred constants here:
+                    // - GCEConsts have been handled separately
+                    // - Repeat expr count back compat consts have also been handled separately
+                    // So we are free to simply defer evaluation here.
+                    //
+                    // FIXME: This assumes that `args` are normalized which is not necessarily true
+                    //
+                    // Const patterns are converted to type system constants before being
+                    // evaluated. However, we don't care about them here as pattern evaluation
+                    // logic does not go through type system normalization. If it did this would
+                    // be a backwards compatibility problem as we do not enforce "syntactic" non-
+                    // usage of generic parameters like we do here.
+                    if uv.args.has_non_region_param() || uv.args.has_non_region_infer() {
+                        return Err(EvaluateConstErr::HasGenericsOrInfers);
+                    }
+
+                    let typing_env = infcx
+                        .typing_env(tcx.erase_regions(param_env))
+                        .with_post_analysis_normalized(tcx);
+                    (uv.args, typing_env)
                 }
-            } else if tcx.def_kind(uv.def) == DefKind::AnonConst && uv.has_non_region_infer() {
-                // FIXME: remove this when `const_evaluatable_unchecked` is a hard error.
-                //
-                // Diagnostics will sometimes replace the identity args of anon consts in
-                // array repeat expr counts with inference variables so we have to handle this
-                // even though it is not something we should ever actually encounter.
-                //
-                // Array repeat expr counts are allowed to syntactically use generic parameters
-                // but must not actually depend on them in order to evalaute successfully. This means
-                // that it is actually fine to evalaute them in their own environment rather than with
-                // the actually provided generic arguments.
-                tcx.dcx().delayed_bug(
-                    "Encountered anon const with inference variable args but no error reported",
-                );
-
-                let args = GenericArgs::identity_for_item(tcx, uv.def);
-                let typing_env = ty::TypingEnv::post_analysis(tcx, uv.def);
-                (args, typing_env)
-            } else {
-                // FIXME: This codepath is reachable under `associated_const_equality` and in the
-                // future will be reachable by `min_generic_const_args`. We should handle inference
-                // variables and generic parameters properly instead of doing nothing.
-                let typing_env = infcx
-                    .typing_env(tcx.erase_regions(param_env))
-                    .with_post_analysis_normalized(tcx);
-                (uv.args, typing_env)
             };
-            let uv = ty::UnevaluatedConst::new(uv.def, args);
 
+            let uv = ty::UnevaluatedConst::new(uv.def, args);
             let erased_uv = tcx.erase_regions(uv);
+
             use rustc_middle::mir::interpret::ErrorHandled;
             match tcx.const_eval_resolve_for_typeck(typing_env, erased_uv, DUMMY_SP) {
                 Ok(Ok(val)) => Ok(ty::Const::new_value(
diff --git a/compiler/rustc_trait_selection/src/traits/normalize.rs b/compiler/rustc_trait_selection/src/traits/normalize.rs
index 88a0c402702..eb6d5c8a60a 100644
--- a/compiler/rustc_trait_selection/src/traits/normalize.rs
+++ b/compiler/rustc_trait_selection/src/traits/normalize.rs
@@ -299,12 +299,21 @@ impl<'a, 'b, 'tcx> AssocTypeNormalizer<'a, 'b, 'tcx> {
             );
         }
 
+        // We don't replace bound vars in the generic arguments of the free alias with
+        // placeholders. This doesn't cause any issues as instantiating parameters with
+        // bound variables is special-cased to rewrite the debruijn index to be higher
+        // whenever we fold through a binder.
+        //
+        // However, we do replace any escaping bound vars in the resulting goals with
+        // placeholders as the trait solver does not expect to encounter escaping bound
+        // vars in obligations.
+        //
+        // FIXME(lazy_type_alias): Check how much this actually matters for perf before
+        // stabilization. This is a bit weird and generally not how we handle binders in
+        // the compiler so ideally we'd do the same boundvar->placeholder->boundvar dance
+        // that other kinds of normalization do.
         let infcx = self.selcx.infcx;
         self.obligations.extend(
-            // FIXME(BoxyUwU):
-            // FIXME(lazy_type_alias):
-            // It seems suspicious to instantiate the predicates with arguments that might be bound vars,
-            // we might wind up instantiating one of these bound vars underneath a hrtb.
             infcx.tcx.predicates_of(free.def_id).instantiate_own(infcx.tcx, free.args).map(
                 |(mut predicate, span)| {
                     if free.has_escaping_bound_vars() {
diff --git a/compiler/rustc_trait_selection/src/traits/outlives_bounds.rs b/compiler/rustc_trait_selection/src/traits/outlives_bounds.rs
index 68983ef80fa..59d3ac21387 100644
--- a/compiler/rustc_trait_selection/src/traits/outlives_bounds.rs
+++ b/compiler/rustc_trait_selection/src/traits/outlives_bounds.rs
@@ -9,7 +9,7 @@ use rustc_span::def_id::LocalDefId;
 use tracing::instrument;
 
 use crate::infer::InferCtxt;
-use crate::traits::{ObligationCause, ObligationCtxt};
+use crate::traits::ObligationCause;
 
 /// Implied bounds are region relationships that we deduce
 /// automatically. The idea is that (e.g.) a caller must check that a
@@ -79,24 +79,9 @@ fn implied_outlives_bounds<'a, 'tcx>(
 
     if !constraints.is_empty() {
         let QueryRegionConstraints { outlives } = constraints;
-        // Instantiation may have produced new inference variables and constraints on those
-        // variables. Process these constraints.
-        let ocx = ObligationCtxt::new(infcx);
         let cause = ObligationCause::misc(span, body_id);
-        for &constraint in &outlives {
-            ocx.register_obligation(infcx.query_outlives_constraint_to_obligation(
-                constraint,
-                cause.clone(),
-                param_env,
-            ));
-        }
-
-        let errors = ocx.select_all_or_error();
-        if !errors.is_empty() {
-            infcx.dcx().span_bug(
-                span,
-                "implied_outlives_bounds failed to solve obligations from instantiation",
-            );
+        for &(predicate, _) in &outlives {
+            infcx.register_outlives_constraint(predicate, &cause);
         }
     };
 
diff --git a/compiler/rustc_trait_selection/src/traits/project.rs b/compiler/rustc_trait_selection/src/traits/project.rs
index ca58da5ca6d..6dd80551980 100644
--- a/compiler/rustc_trait_selection/src/traits/project.rs
+++ b/compiler/rustc_trait_selection/src/traits/project.rs
@@ -378,6 +378,7 @@ pub(super) fn opt_normalize_projection_term<'a, 'b, 'tcx>(
             term: projected_term,
             obligations: mut projected_obligations,
         })) => {
+            debug!("opt_normalize_projection_type: progress");
             // if projection succeeded, then what we get out of this
             // is also non-normalized (consider: it was derived from
             // an impl, where-clause etc) and hence we must
@@ -408,6 +409,7 @@ pub(super) fn opt_normalize_projection_term<'a, 'b, 'tcx>(
             Ok(Some(result.value))
         }
         Ok(Projected::NoProgress(projected_ty)) => {
+            debug!("opt_normalize_projection_type: no progress");
             let result =
                 Normalized { value: projected_ty, obligations: PredicateObligations::new() };
             infcx.inner.borrow_mut().projection_cache().insert_term(cache_key, result.clone());
@@ -621,8 +623,17 @@ struct Progress<'tcx> {
 }
 
 impl<'tcx> Progress<'tcx> {
-    fn error(tcx: TyCtxt<'tcx>, guar: ErrorGuaranteed) -> Self {
-        Progress { term: Ty::new_error(tcx, guar).into(), obligations: PredicateObligations::new() }
+    fn error_for_term(
+        tcx: TyCtxt<'tcx>,
+        alias_term: ty::AliasTerm<'tcx>,
+        guar: ErrorGuaranteed,
+    ) -> Self {
+        let err_term = if alias_term.kind(tcx).is_type() {
+            Ty::new_error(tcx, guar).into()
+        } else {
+            ty::Const::new_error(tcx, guar).into()
+        };
+        Progress { term: err_term, obligations: PredicateObligations::new() }
     }
 
     fn with_addl_obligations(mut self, mut obligations: PredicateObligations<'tcx>) -> Self {
@@ -650,7 +661,11 @@ fn project<'cx, 'tcx>(
     }
 
     if let Err(guar) = obligation.predicate.error_reported() {
-        return Ok(Projected::Progress(Progress::error(selcx.tcx(), guar)));
+        return Ok(Projected::Progress(Progress::error_for_term(
+            selcx.tcx(),
+            obligation.predicate,
+            guar,
+        )));
     }
 
     let mut candidates = ProjectionCandidateSet::None;
@@ -1102,7 +1117,7 @@ fn assemble_candidates_from_impls<'cx, 'tcx>(
                                                 selcx.tcx(),
                                                 selcx.tcx().require_lang_item(
                                                     LangItem::Sized,
-                                                    Some(obligation.cause.span),
+                                                    obligation.cause.span,
                                                 ),
                                                 [self_ty],
                                             ),
@@ -1302,7 +1317,7 @@ fn confirm_coroutine_candidate<'cx, 'tcx>(
 
     let tcx = selcx.tcx();
 
-    let coroutine_def_id = tcx.require_lang_item(LangItem::Coroutine, None);
+    let coroutine_def_id = tcx.require_lang_item(LangItem::Coroutine, obligation.cause.span);
 
     let (trait_ref, yield_ty, return_ty) = super::util::coroutine_trait_ref_and_outputs(
         tcx,
@@ -1360,7 +1375,7 @@ fn confirm_future_candidate<'cx, 'tcx>(
     debug!(?obligation, ?coroutine_sig, ?obligations, "confirm_future_candidate");
 
     let tcx = selcx.tcx();
-    let fut_def_id = tcx.require_lang_item(LangItem::Future, None);
+    let fut_def_id = tcx.require_lang_item(LangItem::Future, obligation.cause.span);
 
     let (trait_ref, return_ty) = super::util::future_trait_ref_and_outputs(
         tcx,
@@ -1406,7 +1421,7 @@ fn confirm_iterator_candidate<'cx, 'tcx>(
     debug!(?obligation, ?gen_sig, ?obligations, "confirm_iterator_candidate");
 
     let tcx = selcx.tcx();
-    let iter_def_id = tcx.require_lang_item(LangItem::Iterator, None);
+    let iter_def_id = tcx.require_lang_item(LangItem::Iterator, obligation.cause.span);
 
     let (trait_ref, yield_ty) = super::util::iterator_trait_ref_and_outputs(
         tcx,
@@ -1452,7 +1467,7 @@ fn confirm_async_iterator_candidate<'cx, 'tcx>(
     debug!(?obligation, ?gen_sig, ?obligations, "confirm_async_iterator_candidate");
 
     let tcx = selcx.tcx();
-    let iter_def_id = tcx.require_lang_item(LangItem::AsyncIterator, None);
+    let iter_def_id = tcx.require_lang_item(LangItem::AsyncIterator, obligation.cause.span);
 
     let (trait_ref, yield_ty) = super::util::async_iterator_trait_ref_and_outputs(
         tcx,
@@ -1496,12 +1511,13 @@ fn confirm_builtin_candidate<'cx, 'tcx>(
     let trait_def_id = tcx.trait_of_item(item_def_id).unwrap();
     let args = tcx.mk_args(&[self_ty.into()]);
     let (term, obligations) = if tcx.is_lang_item(trait_def_id, LangItem::DiscriminantKind) {
-        let discriminant_def_id = tcx.require_lang_item(LangItem::Discriminant, None);
+        let discriminant_def_id =
+            tcx.require_lang_item(LangItem::Discriminant, obligation.cause.span);
         assert_eq!(discriminant_def_id, item_def_id);
 
         (self_ty.discriminant_ty(tcx).into(), PredicateObligations::new())
     } else if tcx.is_lang_item(trait_def_id, LangItem::PointeeTrait) {
-        let metadata_def_id = tcx.require_lang_item(LangItem::Metadata, None);
+        let metadata_def_id = tcx.require_lang_item(LangItem::Metadata, obligation.cause.span);
         assert_eq!(metadata_def_id, item_def_id);
 
         let mut obligations = PredicateObligations::new();
@@ -1523,7 +1539,7 @@ fn confirm_builtin_candidate<'cx, 'tcx>(
                 // exist. Instead, `Pointee<Metadata = ()>` should be a supertrait of `Sized`.
                 let sized_predicate = ty::TraitRef::new(
                     tcx,
-                    tcx.require_lang_item(LangItem::Sized, Some(obligation.cause.span)),
+                    tcx.require_lang_item(LangItem::Sized, obligation.cause.span),
                     [self_ty],
                 );
                 obligations.push(obligation.with(tcx, sized_predicate));
@@ -1605,7 +1621,7 @@ fn confirm_closure_candidate<'cx, 'tcx>(
                     )
                 } else {
                     let upvars_projection_def_id =
-                        tcx.require_lang_item(LangItem::AsyncFnKindUpvars, None);
+                        tcx.require_lang_item(LangItem::AsyncFnKindUpvars, obligation.cause.span);
                     let tupled_upvars_ty = Ty::new_projection(
                         tcx,
                         upvars_projection_def_id,
@@ -1666,8 +1682,9 @@ fn confirm_callable_candidate<'cx, 'tcx>(
 
     debug!(?obligation, ?fn_sig, "confirm_callable_candidate");
 
-    let fn_once_def_id = tcx.require_lang_item(LangItem::FnOnce, None);
-    let fn_once_output_def_id = tcx.require_lang_item(LangItem::FnOnceOutput, None);
+    let fn_once_def_id = tcx.require_lang_item(LangItem::FnOnce, obligation.cause.span);
+    let fn_once_output_def_id =
+        tcx.require_lang_item(LangItem::FnOnceOutput, obligation.cause.span);
 
     let predicate = super::util::closure_trait_ref_and_return_type(
         tcx,
@@ -1725,8 +1742,8 @@ fn confirm_async_closure_candidate<'cx, 'tcx>(
                             args.coroutine_captures_by_ref_ty(),
                         )
                     } else {
-                        let upvars_projection_def_id =
-                            tcx.require_lang_item(LangItem::AsyncFnKindUpvars, None);
+                        let upvars_projection_def_id = tcx
+                            .require_lang_item(LangItem::AsyncFnKindUpvars, obligation.cause.span);
                         // When we don't know the closure kind (and therefore also the closure's upvars,
                         // which are computed at the same time), we must delay the computation of the
                         // generator's upvars. We do this using the `AsyncFnKindHelper`, which as a trait
@@ -1783,7 +1800,8 @@ fn confirm_async_closure_candidate<'cx, 'tcx>(
             let term = match item_name {
                 sym::CallOnceFuture | sym::CallRefFuture => sig.output(),
                 sym::Output => {
-                    let future_output_def_id = tcx.require_lang_item(LangItem::FutureOutput, None);
+                    let future_output_def_id =
+                        tcx.require_lang_item(LangItem::FutureOutput, obligation.cause.span);
                     Ty::new_projection(tcx, future_output_def_id, [sig.output()])
                 }
                 name => bug!("no such associated type: {name}"),
@@ -1816,7 +1834,8 @@ fn confirm_async_closure_candidate<'cx, 'tcx>(
             let term = match item_name {
                 sym::CallOnceFuture | sym::CallRefFuture => sig.output(),
                 sym::Output => {
-                    let future_output_def_id = tcx.require_lang_item(LangItem::FutureOutput, None);
+                    let future_output_def_id =
+                        tcx.require_lang_item(LangItem::FutureOutput, obligation.cause.span);
                     Ty::new_projection(tcx, future_output_def_id, [sig.output()])
                 }
                 name => bug!("no such associated type: {name}"),
@@ -1965,7 +1984,13 @@ fn confirm_impl_candidate<'cx, 'tcx>(
     let param_env = obligation.param_env;
     let assoc_term = match specialization_graph::assoc_def(tcx, impl_def_id, assoc_item_id) {
         Ok(assoc_term) => assoc_term,
-        Err(guar) => return Ok(Projected::Progress(Progress::error(tcx, guar))),
+        Err(guar) => {
+            return Ok(Projected::Progress(Progress::error_for_term(
+                tcx,
+                obligation.predicate,
+                guar,
+            )));
+        }
     };
 
     // This means that the impl is missing a definition for the
diff --git a/compiler/rustc_trait_selection/src/traits/query/type_op/implied_outlives_bounds.rs b/compiler/rustc_trait_selection/src/traits/query/type_op/implied_outlives_bounds.rs
index d9b57f0c67d..e294f7839aa 100644
--- a/compiler/rustc_trait_selection/src/traits/query/type_op/implied_outlives_bounds.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/type_op/implied_outlives_bounds.rs
@@ -1,6 +1,6 @@
 use std::ops::ControlFlow;
 
-use rustc_infer::infer::RegionObligation;
+use rustc_infer::infer::TypeOutlivesConstraint;
 use rustc_infer::infer::canonical::CanonicalQueryInput;
 use rustc_infer::traits::query::OutlivesBound;
 use rustc_infer::traits::query::type_op::ImpliedOutlivesBounds;
@@ -141,7 +141,7 @@ pub fn compute_implied_outlives_bounds_inner<'tcx>(
         && !ocx.infcx.tcx.sess.opts.unstable_opts.no_implied_bounds_compat
         && ty.visit_with(&mut ContainsBevyParamSet { tcx: ocx.infcx.tcx }).is_break()
     {
-        for RegionObligation { sup_type, sub_region, .. } in
+        for TypeOutlivesConstraint { sup_type, sub_region, .. } in
             ocx.infcx.take_registered_region_obligations()
         {
             let mut components = smallvec![];
diff --git a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
index 10a2ba049d8..97ecf9702e6 100644
--- a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
@@ -11,7 +11,7 @@ use std::ops::ControlFlow;
 use hir::LangItem;
 use hir::def_id::DefId;
 use rustc_data_structures::fx::{FxHashSet, FxIndexSet};
-use rustc_hir as hir;
+use rustc_hir::{self as hir, CoroutineDesugaring, CoroutineKind};
 use rustc_infer::traits::{Obligation, PolyTraitObligation, SelectionError};
 use rustc_middle::ty::fast_reject::DeepRejectCtxt;
 use rustc_middle::ty::{self, Ty, TypeVisitableExt, TypingMode, elaborate};
@@ -65,71 +65,92 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             let def_id = obligation.predicate.def_id();
             let tcx = self.tcx();
 
-            if tcx.is_lang_item(def_id, LangItem::Copy) {
-                debug!(obligation_self_ty = ?obligation.predicate.skip_binder().self_ty());
+            let lang_item = tcx.as_lang_item(def_id);
+            match lang_item {
+                Some(LangItem::Copy | LangItem::Clone) => {
+                    debug!(obligation_self_ty = ?obligation.predicate.skip_binder().self_ty());
 
-                // User-defined copy impls are permitted, but only for
-                // structs and enums.
-                self.assemble_candidates_from_impls(obligation, &mut candidates);
+                    // User-defined copy impls are permitted, but only for
+                    // structs and enums.
+                    self.assemble_candidates_from_impls(obligation, &mut candidates);
 
-                // For other types, we'll use the builtin rules.
-                let copy_conditions = self.copy_clone_conditions(obligation);
-                self.assemble_builtin_bound_candidates(copy_conditions, &mut candidates);
-            } else if tcx.is_lang_item(def_id, LangItem::DiscriminantKind) {
-                // `DiscriminantKind` is automatically implemented for every type.
-                candidates.vec.push(BuiltinCandidate { has_nested: false });
-            } else if tcx.is_lang_item(def_id, LangItem::PointeeTrait) {
-                // `Pointee` is automatically implemented for every type.
-                candidates.vec.push(BuiltinCandidate { has_nested: false });
-            } else if tcx.is_lang_item(def_id, LangItem::Sized) {
-                self.assemble_builtin_sized_candidate(obligation, &mut candidates);
-            } else if tcx.is_lang_item(def_id, LangItem::Unsize) {
-                self.assemble_candidates_for_unsizing(obligation, &mut candidates);
-            } else if tcx.is_lang_item(def_id, LangItem::Destruct) {
-                self.assemble_const_destruct_candidates(obligation, &mut candidates);
-            } else if tcx.is_lang_item(def_id, LangItem::TransmuteTrait) {
-                // User-defined transmutability impls are permitted.
-                self.assemble_candidates_from_impls(obligation, &mut candidates);
-                self.assemble_candidates_for_transmutability(obligation, &mut candidates);
-            } else if tcx.is_lang_item(def_id, LangItem::Tuple) {
-                self.assemble_candidate_for_tuple(obligation, &mut candidates);
-            } else if tcx.is_lang_item(def_id, LangItem::FnPtrTrait) {
-                self.assemble_candidates_for_fn_ptr_trait(obligation, &mut candidates);
-            } else if tcx.is_lang_item(def_id, LangItem::BikeshedGuaranteedNoDrop) {
-                self.assemble_candidates_for_bikeshed_guaranteed_no_drop_trait(
-                    obligation,
-                    &mut candidates,
-                );
-            } else {
-                if tcx.is_lang_item(def_id, LangItem::Clone) {
-                    // Same builtin conditions as `Copy`, i.e., every type which has builtin support
-                    // for `Copy` also has builtin support for `Clone`, and tuples/arrays of `Clone`
-                    // types have builtin support for `Clone`.
-                    let clone_conditions = self.copy_clone_conditions(obligation);
-                    self.assemble_builtin_bound_candidates(clone_conditions, &mut candidates);
+                    // For other types, we'll use the builtin rules.
+                    let copy_conditions = self.copy_clone_conditions(obligation);
+                    self.assemble_builtin_bound_candidates(copy_conditions, &mut candidates);
                 }
-
-                if tcx.is_lang_item(def_id, LangItem::Coroutine) {
-                    self.assemble_coroutine_candidates(obligation, &mut candidates);
-                } else if tcx.is_lang_item(def_id, LangItem::Future) {
-                    self.assemble_future_candidates(obligation, &mut candidates);
-                } else if tcx.is_lang_item(def_id, LangItem::Iterator) {
-                    self.assemble_iterator_candidates(obligation, &mut candidates);
-                } else if tcx.is_lang_item(def_id, LangItem::FusedIterator) {
-                    self.assemble_fused_iterator_candidates(obligation, &mut candidates);
-                } else if tcx.is_lang_item(def_id, LangItem::AsyncIterator) {
-                    self.assemble_async_iterator_candidates(obligation, &mut candidates);
-                } else if tcx.is_lang_item(def_id, LangItem::AsyncFnKindHelper) {
-                    self.assemble_async_fn_kind_helper_candidates(obligation, &mut candidates);
+                Some(LangItem::DiscriminantKind) => {
+                    // `DiscriminantKind` is automatically implemented for every type.
+                    candidates.vec.push(BuiltinCandidate { has_nested: false });
                 }
+                Some(LangItem::PointeeTrait) => {
+                    // `Pointee` is automatically implemented for every type.
+                    candidates.vec.push(BuiltinCandidate { has_nested: false });
+                }
+                Some(LangItem::Sized) => {
+                    self.assemble_builtin_sized_candidate(obligation, &mut candidates);
+                }
+                Some(LangItem::Unsize) => {
+                    self.assemble_candidates_for_unsizing(obligation, &mut candidates);
+                }
+                Some(LangItem::Destruct) => {
+                    self.assemble_const_destruct_candidates(obligation, &mut candidates);
+                }
+                Some(LangItem::TransmuteTrait) => {
+                    // User-defined transmutability impls are permitted.
+                    self.assemble_candidates_from_impls(obligation, &mut candidates);
+                    self.assemble_candidates_for_transmutability(obligation, &mut candidates);
+                }
+                Some(LangItem::Tuple) => {
+                    self.assemble_candidate_for_tuple(obligation, &mut candidates);
+                }
+                Some(LangItem::FnPtrTrait) => {
+                    self.assemble_candidates_for_fn_ptr_trait(obligation, &mut candidates);
+                }
+                Some(LangItem::BikeshedGuaranteedNoDrop) => {
+                    self.assemble_candidates_for_bikeshed_guaranteed_no_drop_trait(
+                        obligation,
+                        &mut candidates,
+                    );
+                }
+                _ => {
+                    // We re-match here for traits that can have both builtin impls and user written impls.
+                    // After the builtin impls we need to also add user written impls, which we do not want to
+                    // do in general because just checking if there are any is expensive.
+                    match lang_item {
+                        Some(LangItem::Coroutine) => {
+                            self.assemble_coroutine_candidates(obligation, &mut candidates);
+                        }
+                        Some(LangItem::Future) => {
+                            self.assemble_future_candidates(obligation, &mut candidates);
+                        }
+                        Some(LangItem::Iterator) => {
+                            self.assemble_iterator_candidates(obligation, &mut candidates);
+                        }
+                        Some(LangItem::FusedIterator) => {
+                            self.assemble_fused_iterator_candidates(obligation, &mut candidates);
+                        }
+                        Some(LangItem::AsyncIterator) => {
+                            self.assemble_async_iterator_candidates(obligation, &mut candidates);
+                        }
+                        Some(LangItem::AsyncFnKindHelper) => {
+                            self.assemble_async_fn_kind_helper_candidates(
+                                obligation,
+                                &mut candidates,
+                            );
+                        }
+                        Some(LangItem::AsyncFn | LangItem::AsyncFnMut | LangItem::AsyncFnOnce) => {
+                            self.assemble_async_closure_candidates(obligation, &mut candidates);
+                        }
+                        Some(LangItem::Fn | LangItem::FnMut | LangItem::FnOnce) => {
+                            self.assemble_closure_candidates(obligation, &mut candidates);
+                            self.assemble_fn_pointer_candidates(obligation, &mut candidates);
+                        }
+                        _ => {}
+                    }
 
-                // FIXME: Put these into `else if` blocks above, since they're built-in.
-                self.assemble_closure_candidates(obligation, &mut candidates);
-                self.assemble_async_closure_candidates(obligation, &mut candidates);
-                self.assemble_fn_pointer_candidates(obligation, &mut candidates);
-
-                self.assemble_candidates_from_impls(obligation, &mut candidates);
-                self.assemble_candidates_from_object_ty(obligation, &mut candidates);
+                    self.assemble_candidates_from_impls(obligation, &mut candidates);
+                    self.assemble_candidates_from_object_ty(obligation, &mut candidates);
+                }
             }
 
             self.assemble_candidates_from_projected_tys(obligation, &mut candidates);
@@ -360,9 +381,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         obligation: &PolyTraitObligation<'tcx>,
         candidates: &mut SelectionCandidateSet<'tcx>,
     ) {
-        let Some(kind) = self.tcx().fn_trait_kind_from_def_id(obligation.predicate.def_id()) else {
-            return;
-        };
+        let kind = self.tcx().fn_trait_kind_from_def_id(obligation.predicate.def_id()).unwrap();
 
         // Okay to skip binder because the args on closure types never
         // touch bound regions, they just capture the in-scope
@@ -419,26 +438,39 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         }
     }
 
+    #[instrument(level = "debug", skip(self, candidates))]
     fn assemble_async_closure_candidates(
         &mut self,
         obligation: &PolyTraitObligation<'tcx>,
         candidates: &mut SelectionCandidateSet<'tcx>,
     ) {
-        let Some(goal_kind) =
-            self.tcx().async_fn_trait_kind_from_def_id(obligation.predicate.def_id())
-        else {
-            return;
-        };
+        let goal_kind =
+            self.tcx().async_fn_trait_kind_from_def_id(obligation.predicate.def_id()).unwrap();
 
+        debug!("self_ty = {:?}", obligation.self_ty().skip_binder().kind());
         match *obligation.self_ty().skip_binder().kind() {
-            ty::CoroutineClosure(_, args) => {
+            ty::CoroutineClosure(def_id, args) => {
                 if let Some(closure_kind) =
                     args.as_coroutine_closure().kind_ty().to_opt_closure_kind()
                     && !closure_kind.extends(goal_kind)
                 {
                     return;
                 }
-                candidates.vec.push(AsyncClosureCandidate);
+
+                // Make sure this is actually an async closure.
+                let Some(coroutine_kind) =
+                    self.tcx().coroutine_kind(self.tcx().coroutine_for_closure(def_id))
+                else {
+                    bug!("coroutine with no kind");
+                };
+
+                debug!(?coroutine_kind);
+                match coroutine_kind {
+                    CoroutineKind::Desugared(CoroutineDesugaring::Async, _) => {
+                        candidates.vec.push(AsyncClosureCandidate);
+                    }
+                    _ => (),
+                }
             }
             // Closures and fn pointers implement `AsyncFn*` if their return types
             // implement `Future`, which is checked later.
@@ -501,11 +533,6 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         obligation: &PolyTraitObligation<'tcx>,
         candidates: &mut SelectionCandidateSet<'tcx>,
     ) {
-        // We provide impl of all fn traits for fn pointers.
-        if !self.tcx().is_fn_trait(obligation.predicate.def_id()) {
-            return;
-        }
-
         // Keep this function in sync with extract_tupled_inputs_and_output_from_callable
         // until the old solver (and thus this function) is removed.
 
diff --git a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
index 94190cd3ae3..7acf0f990d1 100644
--- a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
@@ -251,16 +251,11 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         let tcx = self.tcx();
         let obligations = if has_nested {
             let trait_def = obligation.predicate.def_id();
-            let conditions = if tcx.is_lang_item(trait_def, LangItem::Sized) {
-                self.sized_conditions(obligation)
-            } else if tcx.is_lang_item(trait_def, LangItem::Copy) {
-                self.copy_clone_conditions(obligation)
-            } else if tcx.is_lang_item(trait_def, LangItem::Clone) {
-                self.copy_clone_conditions(obligation)
-            } else if tcx.is_lang_item(trait_def, LangItem::FusedIterator) {
-                self.fused_iterator_conditions(obligation)
-            } else {
-                bug!("unexpected builtin trait {:?}", trait_def)
+            let conditions = match tcx.as_lang_item(trait_def) {
+                Some(LangItem::Sized) => self.sized_conditions(obligation),
+                Some(LangItem::Copy | LangItem::Clone) => self.copy_clone_conditions(obligation),
+                Some(LangItem::FusedIterator) => self.fused_iterator_conditions(obligation),
+                other => bug!("unexpected builtin trait {trait_def:?} ({other:?})"),
             };
             let BuiltinImplConditions::Where(types) = conditions else {
                 bug!("obligation {:?} had matched a builtin impl but now doesn't", obligation);
@@ -323,7 +318,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             let make_freeze_obl = |ty| {
                 let trait_ref = ty::TraitRef::new(
                     tcx,
-                    tcx.require_lang_item(LangItem::Freeze, None),
+                    tcx.require_lang_item(LangItem::Freeze, obligation.cause.span),
                     [ty::GenericArg::from(ty)],
                 );
                 Obligation::with_depth(
@@ -662,7 +657,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         );
         let tr = ty::TraitRef::new(
             self.tcx(),
-            self.tcx().require_lang_item(LangItem::Sized, Some(cause.span)),
+            self.tcx().require_lang_item(LangItem::Sized, cause.span),
             [output_ty],
         );
         nested.push(Obligation::new(self.infcx.tcx, cause, obligation.param_env, tr));
@@ -882,14 +877,16 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 });
 
                 // We must additionally check that the return type impls `Future + Sized`.
-                let future_trait_def_id = tcx.require_lang_item(LangItem::Future, None);
+                let future_trait_def_id =
+                    tcx.require_lang_item(LangItem::Future, obligation.cause.span);
                 nested.push(obligation.with(
                     tcx,
                     sig.output().map_bound(|output_ty| {
                         ty::TraitRef::new(tcx, future_trait_def_id, [output_ty])
                     }),
                 ));
-                let sized_trait_def_id = tcx.require_lang_item(LangItem::Sized, None);
+                let sized_trait_def_id =
+                    tcx.require_lang_item(LangItem::Sized, obligation.cause.span);
                 nested.push(obligation.with(
                     tcx,
                     sig.output().map_bound(|output_ty| {
@@ -911,13 +908,15 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 });
 
                 // We must additionally check that the return type impls `Future + Sized`.
-                let future_trait_def_id = tcx.require_lang_item(LangItem::Future, None);
+                let future_trait_def_id =
+                    tcx.require_lang_item(LangItem::Future, obligation.cause.span);
                 let placeholder_output_ty = self.infcx.enter_forall_and_leak_universe(sig.output());
                 nested.push(obligation.with(
                     tcx,
                     ty::TraitRef::new(tcx, future_trait_def_id, [placeholder_output_ty]),
                 ));
-                let sized_trait_def_id = tcx.require_lang_item(LangItem::Sized, None);
+                let sized_trait_def_id =
+                    tcx.require_lang_item(LangItem::Sized, obligation.cause.span);
                 nested.push(obligation.with(
                     tcx,
                     sig.output().map_bound(|output_ty| {
@@ -951,10 +950,8 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 obligation.param_env,
                 ty::TraitRef::new(
                     self.tcx(),
-                    self.tcx().require_lang_item(
-                        LangItem::AsyncFnKindHelper,
-                        Some(obligation.cause.span),
-                    ),
+                    self.tcx()
+                        .require_lang_item(LangItem::AsyncFnKindHelper, obligation.cause.span),
                     [kind_ty, Ty::from_closure_kind(self.tcx(), goal_kind)],
                 ),
             ));
@@ -1170,7 +1167,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 // We can only make objects from sized types.
                 let tr = ty::TraitRef::new(
                     tcx,
-                    tcx.require_lang_item(LangItem::Sized, Some(obligation.cause.span)),
+                    tcx.require_lang_item(LangItem::Sized, obligation.cause.span),
                     [source],
                 );
                 nested.push(predicate_to_obligation(tr.upcast(tcx)));
@@ -1364,7 +1361,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     self_ty.map_bound(|ty| {
                         ty::TraitRef::new(
                             tcx,
-                            tcx.require_lang_item(LangItem::Copy, Some(obligation.cause.span)),
+                            tcx.require_lang_item(LangItem::Copy, obligation.cause.span),
                             [ty],
                         )
                     }),
@@ -1416,7 +1413,7 @@ fn pointer_like_goal_for_rpitit<'tcx>(
     ty::Binder::bind_with_vars(
         ty::TraitRef::new(
             tcx,
-            tcx.require_lang_item(LangItem::PointerLike, Some(cause.span)),
+            tcx.require_lang_item(LangItem::PointerLike, cause.span),
             [Ty::new_projection_from_args(tcx, rpitit_item, args)],
         ),
         tcx.mk_bound_variable_kinds(&bound_vars),
diff --git a/compiler/rustc_trait_selection/src/traits/select/mod.rs b/compiler/rustc_trait_selection/src/traits/select/mod.rs
index 44a76f6e083..3a2f9e8ca17 100644
--- a/compiler/rustc_trait_selection/src/traits/select/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/mod.rs
@@ -265,9 +265,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         &mut self,
         obligation: &PolyTraitObligation<'tcx>,
     ) -> SelectionResult<'tcx, Selection<'tcx>> {
-        if self.infcx.next_trait_solver() {
-            return self.infcx.select_in_new_trait_solver(obligation);
-        }
+        assert!(!self.infcx.next_trait_solver());
 
         let candidate = match self.select_from_obligation(obligation) {
             Err(SelectionError::Overflow(OverflowError::Canonical)) => {
@@ -299,6 +297,10 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         &mut self,
         obligation: &TraitObligation<'tcx>,
     ) -> SelectionResult<'tcx, Selection<'tcx>> {
+        if self.infcx.next_trait_solver() {
+            return self.infcx.select_in_new_trait_solver(obligation);
+        }
+
         self.poly_select(&Obligation {
             cause: obligation.cause.clone(),
             param_env: obligation.param_env,
@@ -1782,7 +1784,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             if !generics.is_own_empty()
                 && obligation.predicate.args[generics.parent_count..].iter().any(|&p| {
                     p.has_non_region_infer()
-                        && match p.unpack() {
+                        && match p.kind() {
                             ty::GenericArgKind::Const(ct) => {
                                 self.infcx.shallow_resolve_const(ct) != ct
                             }
@@ -2866,7 +2868,7 @@ fn rebind_coroutine_witness_types<'tcx>(
     let shifted_coroutine_types =
         tcx.shift_bound_var_indices(bound_vars.len(), bound_coroutine_types.skip_binder());
     ty::Binder::bind_with_vars(
-        ty::EarlyBinder::bind(shifted_coroutine_types.to_vec()).instantiate(tcx, args),
+        ty::EarlyBinder::bind(shifted_coroutine_types.types.to_vec()).instantiate(tcx, args),
         tcx.mk_bound_variable_kinds_from_iter(
             bound_vars.iter().chain(bound_coroutine_types.bound_vars()),
         ),
diff --git a/compiler/rustc_trait_selection/src/traits/structural_normalize.rs b/compiler/rustc_trait_selection/src/traits/structural_normalize.rs
index e6d5d336b8d..2e20ede2f50 100644
--- a/compiler/rustc_trait_selection/src/traits/structural_normalize.rs
+++ b/compiler/rustc_trait_selection/src/traits/structural_normalize.rs
@@ -39,10 +39,7 @@ impl<'tcx> At<'_, 'tcx> {
                 return Ok(term);
             }
 
-            let new_infer = match term.unpack() {
-                ty::TermKind::Ty(_) => self.infcx.next_ty_var(self.cause.span).into(),
-                ty::TermKind::Const(_) => self.infcx.next_const_var(self.cause.span).into(),
-            };
+            let new_infer = self.infcx.next_term_var_of_kind(term, self.cause.span);
 
             // We simply emit an `alias-eq` goal here, since that will take care of
             // normalizing the LHS of the projection until it is a rigid projection
diff --git a/compiler/rustc_trait_selection/src/traits/wf.rs b/compiler/rustc_trait_selection/src/traits/wf.rs
index 08d3b92e9b5..416865e861e 100644
--- a/compiler/rustc_trait_selection/src/traits/wf.rs
+++ b/compiler/rustc_trait_selection/src/traits/wf.rs
@@ -37,7 +37,7 @@ pub fn obligations<'tcx>(
     span: Span,
 ) -> Option<PredicateObligations<'tcx>> {
     // Handle the "cycle" case (see comment above) by bailing out if necessary.
-    let term = match term.unpack() {
+    let term = match term.kind() {
         TermKind::Ty(ty) => {
             match ty.kind() {
                 ty::Infer(ty::TyVar(_)) => {
@@ -541,7 +541,7 @@ impl<'a, 'tcx> WfPredicates<'a, 'tcx> {
             let cause = self.cause(cause);
             let trait_ref = ty::TraitRef::new(
                 self.tcx(),
-                self.tcx().require_lang_item(LangItem::Sized, Some(cause.span)),
+                self.tcx().require_lang_item(LangItem::Sized, cause.span),
                 [subty],
             );
             self.out.push(traits::Obligation::with_depth(
@@ -895,7 +895,7 @@ impl<'a, 'tcx> TypeVisitor<TyCtxt<'tcx>> for WfPredicates<'a, 'tcx> {
                                 self.tcx(),
                                 self.tcx().require_lang_item(
                                     LangItem::BikeshedGuaranteedNoDrop,
-                                    Some(self.span),
+                                    self.span,
                                 ),
                                 [ty],
                             )
diff --git a/compiler/rustc_traits/src/coroutine_witnesses.rs b/compiler/rustc_traits/src/coroutine_witnesses.rs
new file mode 100644
index 00000000000..447e13126cc
--- /dev/null
+++ b/compiler/rustc_traits/src/coroutine_witnesses.rs
@@ -0,0 +1,37 @@
+use rustc_hir::def_id::DefId;
+use rustc_middle::ty::{self, TyCtxt, fold_regions};
+
+/// Return the set of types that should be taken into account when checking
+/// trait bounds on a coroutine's internal state. This properly replaces
+/// `ReErased` with new existential bound lifetimes.
+pub(crate) fn coroutine_hidden_types<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    def_id: DefId,
+) -> ty::EarlyBinder<'tcx, ty::Binder<'tcx, ty::CoroutineWitnessTypes<TyCtxt<'tcx>>>> {
+    let coroutine_layout = tcx.mir_coroutine_witnesses(def_id);
+    let mut vars = vec![];
+    let bound_tys = tcx.mk_type_list_from_iter(
+        coroutine_layout
+            .as_ref()
+            .map_or_else(|| [].iter(), |l| l.field_tys.iter())
+            .filter(|decl| !decl.ignore_for_traits)
+            .map(|decl| {
+                let ty = fold_regions(tcx, decl.ty, |re, debruijn| {
+                    assert_eq!(re, tcx.lifetimes.re_erased);
+                    let var = ty::BoundVar::from_usize(vars.len());
+                    vars.push(ty::BoundVariableKind::Region(ty::BoundRegionKind::Anon));
+                    ty::Region::new_bound(
+                        tcx,
+                        debruijn,
+                        ty::BoundRegion { var, kind: ty::BoundRegionKind::Anon },
+                    )
+                });
+                ty
+            }),
+    );
+
+    ty::EarlyBinder::bind(ty::Binder::bind_with_vars(
+        ty::CoroutineWitnessTypes { types: bound_tys },
+        tcx.mk_bound_variable_kinds(&vars),
+    ))
+}
diff --git a/compiler/rustc_traits/src/lib.rs b/compiler/rustc_traits/src/lib.rs
index 697c8391803..32d8c3f58e0 100644
--- a/compiler/rustc_traits/src/lib.rs
+++ b/compiler/rustc_traits/src/lib.rs
@@ -5,6 +5,7 @@
 // tidy-alphabetical-end
 
 mod codegen;
+mod coroutine_witnesses;
 mod dropck_outlives;
 mod evaluate_obligation;
 mod implied_outlives_bounds;
@@ -24,4 +25,5 @@ pub fn provide(p: &mut Providers) {
     normalize_erasing_regions::provide(p);
     type_op::provide(p);
     p.codegen_select_candidate = codegen::codegen_select_candidate;
+    p.coroutine_hidden_types = coroutine_witnesses::coroutine_hidden_types;
 }
diff --git a/compiler/rustc_transmute/src/layout/tree.rs b/compiler/rustc_transmute/src/layout/tree.rs
index 7cf712ce9e9..ff665695b5a 100644
--- a/compiler/rustc_transmute/src/layout/tree.rs
+++ b/compiler/rustc_transmute/src/layout/tree.rs
@@ -451,7 +451,7 @@ pub(crate) mod rustc {
 
                     // For enums (but not coroutines), the tag field is
                     // currently always the first field of the layout.
-                    assert_eq!(*tag_field, 0);
+                    assert_eq!(*tag_field, FieldIdx::ZERO);
 
                     let variants = def.discriminants(cx.tcx()).try_fold(
                         Self::uninhabited(),
diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs
index 2b49d7ac8b5..bb5187e4f5c 100644
--- a/compiler/rustc_ty_utils/src/abi.rs
+++ b/compiler/rustc_ty_utils/src/abi.rs
@@ -11,9 +11,10 @@ use rustc_middle::ty::layout::{
 };
 use rustc_middle::ty::{self, InstanceKind, Ty, TyCtxt};
 use rustc_session::config::OptLevel;
+use rustc_span::DUMMY_SP;
 use rustc_span::def_id::DefId;
 use rustc_target::callconv::{
-    ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, Conv, FnAbi, PassMode, RiscvInterruptKind,
+    AbiMap, ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, FnAbi, PassMode,
 };
 use tracing::debug;
 
@@ -124,7 +125,7 @@ fn fn_sig_for_fn_abi<'tcx>(
 
             let env_ty = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, ty);
 
-            let pin_did = tcx.require_lang_item(LangItem::Pin, None);
+            let pin_did = tcx.require_lang_item(LangItem::Pin, DUMMY_SP);
             let pin_adt_ref = tcx.adt_def(pin_did);
             let pin_args = tcx.mk_args(&[env_ty.into()]);
             let env_ty = match coroutine_kind {
@@ -149,7 +150,7 @@ fn fn_sig_for_fn_abi<'tcx>(
                     // The signature should be `Future::poll(_, &mut Context<'_>) -> Poll<Output>`
                     assert_eq!(sig.yield_ty, tcx.types.unit);
 
-                    let poll_did = tcx.require_lang_item(LangItem::Poll, None);
+                    let poll_did = tcx.require_lang_item(LangItem::Poll, DUMMY_SP);
                     let poll_adt_ref = tcx.adt_def(poll_did);
                     let poll_args = tcx.mk_args(&[sig.return_ty.into()]);
                     let ret_ty = Ty::new_adt(tcx, poll_adt_ref, poll_args);
@@ -160,7 +161,7 @@ fn fn_sig_for_fn_abi<'tcx>(
                     {
                         if let ty::Adt(resume_ty_adt, _) = sig.resume_ty.kind() {
                             let expected_adt =
-                                tcx.adt_def(tcx.require_lang_item(LangItem::ResumeTy, None));
+                                tcx.adt_def(tcx.require_lang_item(LangItem::ResumeTy, DUMMY_SP));
                             assert_eq!(*resume_ty_adt, expected_adt);
                         } else {
                             panic!("expected `ResumeTy`, found `{:?}`", sig.resume_ty);
@@ -172,7 +173,7 @@ fn fn_sig_for_fn_abi<'tcx>(
                 }
                 hir::CoroutineKind::Desugared(hir::CoroutineDesugaring::Gen, _) => {
                     // The signature should be `Iterator::next(_) -> Option<Yield>`
-                    let option_did = tcx.require_lang_item(LangItem::Option, None);
+                    let option_did = tcx.require_lang_item(LangItem::Option, DUMMY_SP);
                     let option_adt_ref = tcx.adt_def(option_did);
                     let option_args = tcx.mk_args(&[sig.yield_ty.into()]);
                     let ret_ty = Ty::new_adt(tcx, option_adt_ref, option_args);
@@ -196,7 +197,7 @@ fn fn_sig_for_fn_abi<'tcx>(
                     {
                         if let ty::Adt(resume_ty_adt, _) = sig.resume_ty.kind() {
                             let expected_adt =
-                                tcx.adt_def(tcx.require_lang_item(LangItem::ResumeTy, None));
+                                tcx.adt_def(tcx.require_lang_item(LangItem::ResumeTy, DUMMY_SP));
                             assert_eq!(*resume_ty_adt, expected_adt);
                         } else {
                             panic!("expected `ResumeTy`, found `{:?}`", sig.resume_ty);
@@ -208,7 +209,7 @@ fn fn_sig_for_fn_abi<'tcx>(
                 }
                 hir::CoroutineKind::Coroutine(_) => {
                     // The signature should be `Coroutine::resume(_, Resume) -> CoroutineState<Yield, Return>`
-                    let state_did = tcx.require_lang_item(LangItem::CoroutineState, None);
+                    let state_did = tcx.require_lang_item(LangItem::CoroutineState, DUMMY_SP);
                     let state_adt_ref = tcx.adt_def(state_did);
                     let state_args = tcx.mk_args(&[sig.yield_ty.into(), sig.return_ty.into()]);
                     let ret_ty = Ty::new_adt(tcx, state_adt_ref, state_args);
@@ -240,45 +241,6 @@ fn fn_sig_for_fn_abi<'tcx>(
     }
 }
 
-#[inline]
-fn conv_from_spec_abi(tcx: TyCtxt<'_>, abi: ExternAbi, c_variadic: bool) -> Conv {
-    use rustc_abi::ExternAbi::*;
-    match tcx.sess.target.adjust_abi(abi, c_variadic) {
-        Rust | RustCall => Conv::Rust,
-
-        // This is intentionally not using `Conv::Cold`, as that has to preserve
-        // even SIMD registers, which is generally not a good trade-off.
-        RustCold => Conv::PreserveMost,
-
-        // It's the ABI's job to select this, not ours.
-        System { .. } => bug!("system abi should be selected elsewhere"),
-        EfiApi => bug!("eficall abi should be selected elsewhere"),
-
-        Stdcall { .. } => Conv::X86Stdcall,
-        Fastcall { .. } => Conv::X86Fastcall,
-        Vectorcall { .. } => Conv::X86VectorCall,
-        Thiscall { .. } => Conv::X86ThisCall,
-        C { .. } => Conv::C,
-        Unadjusted => Conv::C,
-        Win64 { .. } => Conv::X86_64Win64,
-        SysV64 { .. } => Conv::X86_64SysV,
-        Aapcs { .. } => Conv::ArmAapcs,
-        CCmseNonSecureCall => Conv::CCmseNonSecureCall,
-        CCmseNonSecureEntry => Conv::CCmseNonSecureEntry,
-        PtxKernel => Conv::GpuKernel,
-        Msp430Interrupt => Conv::Msp430Intr,
-        X86Interrupt => Conv::X86Intr,
-        GpuKernel => Conv::GpuKernel,
-        AvrInterrupt => Conv::AvrInterrupt,
-        AvrNonBlockingInterrupt => Conv::AvrNonBlockingInterrupt,
-        RiscvInterruptM => Conv::RiscvInterrupt { kind: RiscvInterruptKind::Machine },
-        RiscvInterruptS => Conv::RiscvInterrupt { kind: RiscvInterruptKind::Supervisor },
-
-        // These API constants ought to be more specific...
-        Cdecl { .. } => Conv::C,
-    }
-}
-
 fn fn_abi_of_fn_ptr<'tcx>(
     tcx: TyCtxt<'tcx>,
     query: ty::PseudoCanonicalInput<'tcx, (ty::PolyFnSig<'tcx>, &'tcx ty::List<Ty<'tcx>>)>,
@@ -529,7 +491,8 @@ fn fn_abi_new_uncached<'tcx>(
     };
     let sig = tcx.normalize_erasing_regions(cx.typing_env, sig);
 
-    let conv = conv_from_spec_abi(cx.tcx(), sig.abi, sig.c_variadic);
+    let abi_map = AbiMap::from_target(&tcx.sess.target);
+    let conv = abi_map.canonize_abi(sig.abi, sig.c_variadic).unwrap();
 
     let mut inputs = sig.inputs();
     let extra_args = if sig.abi == ExternAbi::RustCall {
diff --git a/compiler/rustc_ty_utils/src/common_traits.rs b/compiler/rustc_ty_utils/src/common_traits.rs
index bb2c4172b08..7219f40710e 100644
--- a/compiler/rustc_ty_utils/src/common_traits.rs
+++ b/compiler/rustc_ty_utils/src/common_traits.rs
@@ -4,6 +4,7 @@ use rustc_hir::lang_items::LangItem;
 use rustc_infer::infer::TyCtxtInferExt;
 use rustc_middle::query::Providers;
 use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_span::DUMMY_SP;
 use rustc_trait_selection::traits;
 
 fn is_copy_raw<'tcx>(tcx: TyCtxt<'tcx>, query: ty::PseudoCanonicalInput<'tcx, Ty<'tcx>>) -> bool {
@@ -42,7 +43,7 @@ fn is_item_raw<'tcx>(
     item: LangItem,
 ) -> bool {
     let (infcx, param_env) = tcx.infer_ctxt().build_with_typing_env(query.typing_env);
-    let trait_def_id = tcx.require_lang_item(item, None);
+    let trait_def_id = tcx.require_lang_item(item, DUMMY_SP);
     traits::type_known_to_meet_bound_modulo_regions(&infcx, param_env, query.value, trait_def_id)
 }
 
diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs
index ad57555bd24..9774263e4c9 100644
--- a/compiler/rustc_ty_utils/src/layout.rs
+++ b/compiler/rustc_ty_utils/src/layout.rs
@@ -932,7 +932,7 @@ fn variant_info_for_coroutine<'tcx>(
             // However, if the discriminant is placed past the end of the variant, then we need
             // to factor in the size of the discriminant manually. This really should be refactored
             // better, but this "works" for now.
-            if layout.fields.offset(tag_field) >= variant_size {
+            if layout.fields.offset(tag_field.as_usize()) >= variant_size {
                 variant_size += match tag_encoding {
                     TagEncoding::Direct => tag.size(cx),
                     _ => Size::ZERO,
diff --git a/compiler/rustc_ty_utils/src/opaque_types.rs b/compiler/rustc_ty_utils/src/opaque_types.rs
index 841f602d985..3b4482146d4 100644
--- a/compiler/rustc_ty_utils/src/opaque_types.rs
+++ b/compiler/rustc_ty_utils/src/opaque_types.rs
@@ -321,7 +321,10 @@ fn opaque_types_defined_by<'tcx>(
             collector.collect_taits_declared_in_body();
         }
         // Closures and coroutines are type checked with their parent
-        DefKind::Closure | DefKind::InlineConst => {
+        // Note that we also support `SyntheticCoroutineBody` since we create
+        // a MIR body for the def kind, and some MIR passes (like promotion)
+        // may require doing analysis using its typing env.
+        DefKind::Closure | DefKind::InlineConst | DefKind::SyntheticCoroutineBody => {
             collector.opaques.extend(tcx.opaque_types_defined_by(tcx.local_parent(item)));
         }
         DefKind::AssocTy | DefKind::TyAlias | DefKind::GlobalAsm => {}
@@ -343,8 +346,7 @@ fn opaque_types_defined_by<'tcx>(
         | DefKind::ForeignMod
         | DefKind::Field
         | DefKind::LifetimeParam
-        | DefKind::Impl { .. }
-        | DefKind::SyntheticCoroutineBody => {
+        | DefKind::Impl { .. } => {
             span_bug!(
                 tcx.def_span(item),
                 "`opaque_types_defined_by` not defined for {} `{item:?}`",
diff --git a/compiler/rustc_ty_utils/src/representability.rs b/compiler/rustc_ty_utils/src/representability.rs
index 98b1550e1a3..33d334092ba 100644
--- a/compiler/rustc_ty_utils/src/representability.rs
+++ b/compiler/rustc_ty_utils/src/representability.rs
@@ -74,7 +74,7 @@ fn representability_adt_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Representab
     // but the type parameters may cause a cycle with an upstream type
     let params_in_repr = tcx.params_in_repr(adt.did());
     for (i, arg) in args.iter().enumerate() {
-        if let ty::GenericArgKind::Type(ty) = arg.unpack() {
+        if let ty::GenericArgKind::Type(ty) = arg.kind() {
             if params_in_repr.contains(i as u32) {
                 rtry!(representability_ty(tcx, ty));
             }
@@ -104,7 +104,7 @@ fn params_in_repr_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, params_in_repr: &mut
         ty::Adt(adt, args) => {
             let inner_params_in_repr = tcx.params_in_repr(adt.did());
             for (i, arg) in args.iter().enumerate() {
-                if let ty::GenericArgKind::Type(ty) = arg.unpack() {
+                if let ty::GenericArgKind::Type(ty) = arg.kind() {
                     if inner_params_in_repr.contains(i as u32) {
                         params_in_repr_ty(tcx, ty, params_in_repr);
                     }
diff --git a/compiler/rustc_ty_utils/src/structural_match.rs b/compiler/rustc_ty_utils/src/structural_match.rs
index 0b4efab1d9c..e900264a76c 100644
--- a/compiler/rustc_ty_utils/src/structural_match.rs
+++ b/compiler/rustc_ty_utils/src/structural_match.rs
@@ -16,8 +16,7 @@ fn has_structural_eq_impl<'tcx>(tcx: TyCtxt<'tcx>, adt_ty: Ty<'tcx>) -> bool {
 
     let ocx = ObligationCtxt::new(infcx);
     // require `#[derive(PartialEq)]`
-    let structural_peq_def_id =
-        infcx.tcx.require_lang_item(LangItem::StructuralPeq, Some(cause.span));
+    let structural_peq_def_id = infcx.tcx.require_lang_item(LangItem::StructuralPeq, cause.span);
     ocx.register_bound(cause.clone(), ty::ParamEnv::empty(), adt_ty, structural_peq_def_id);
 
     // We deliberately skip *reporting* fulfillment errors (via
diff --git a/compiler/rustc_ty_utils/src/ty.rs b/compiler/rustc_ty_utils/src/ty.rs
index 0c49ddff39b..e39fd6b947b 100644
--- a/compiler/rustc_ty_utils/src/ty.rs
+++ b/compiler/rustc_ty_utils/src/ty.rs
@@ -104,7 +104,7 @@ fn adt_sized_constraint<'tcx>(
 
     // perf hack: if there is a `constraint_ty: Sized` bound, then we know
     // that the type is sized and do not need to check it on the impl.
-    let sized_trait_def_id = tcx.require_lang_item(LangItem::Sized, None);
+    let sized_trait_def_id = tcx.require_lang_item(LangItem::Sized, DUMMY_SP);
     let predicates = tcx.predicates_of(def.did()).predicates;
     if predicates.iter().any(|(p, _)| {
         p.as_trait_clause().is_some_and(|trait_pred| {
@@ -274,7 +274,7 @@ fn unsizing_params_for_adt<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> DenseBitSe
     let def = tcx.adt_def(def_id);
     let num_params = tcx.generics_of(def_id).count();
 
-    let maybe_unsizing_param_idx = |arg: ty::GenericArg<'tcx>| match arg.unpack() {
+    let maybe_unsizing_param_idx = |arg: ty::GenericArg<'tcx>| match arg.kind() {
         ty::GenericArgKind::Type(ty) => match ty.kind() {
             ty::Param(p) => Some(p.index),
             _ => None,
diff --git a/compiler/rustc_type_ir/src/binder.rs b/compiler/rustc_type_ir/src/binder.rs
index 000cf1e1fd8..55c0a3bba9f 100644
--- a/compiler/rustc_type_ir/src/binder.rs
+++ b/compiler/rustc_type_ir/src/binder.rs
@@ -676,7 +676,7 @@ impl<'a, I: Interner> TypeFolder<I> for ArgFolder<'a, I> {
         // the specialized routine `ty::replace_late_regions()`.
         match r.kind() {
             ty::ReEarlyParam(data) => {
-                let rk = self.args.get(data.index() as usize).map(|k| k.kind());
+                let rk = self.args.get(data.index() as usize).map(|arg| arg.kind());
                 match rk {
                     Some(ty::GenericArgKind::Lifetime(lt)) => self.shift_region_through_binders(lt),
                     Some(other) => self.region_param_expected(data, r, other),
@@ -711,12 +711,20 @@ impl<'a, I: Interner> TypeFolder<I> for ArgFolder<'a, I> {
             c.super_fold_with(self)
         }
     }
+
+    fn fold_predicate(&mut self, p: I::Predicate) -> I::Predicate {
+        if p.has_param() { p.super_fold_with(self) } else { p }
+    }
+
+    fn fold_clauses(&mut self, c: I::Clauses) -> I::Clauses {
+        if c.has_param() { c.super_fold_with(self) } else { c }
+    }
 }
 
 impl<'a, I: Interner> ArgFolder<'a, I> {
     fn ty_for_param(&self, p: I::ParamTy, source_ty: I::Ty) -> I::Ty {
         // Look up the type in the args. It really should be in there.
-        let opt_ty = self.args.get(p.index() as usize).map(|k| k.kind());
+        let opt_ty = self.args.get(p.index() as usize).map(|arg| arg.kind());
         let ty = match opt_ty {
             Some(ty::GenericArgKind::Type(ty)) => ty,
             Some(kind) => self.type_param_expected(p, source_ty, kind),
@@ -753,7 +761,7 @@ impl<'a, I: Interner> ArgFolder<'a, I> {
 
     fn const_for_param(&self, p: I::ParamConst, source_ct: I::Const) -> I::Const {
         // Look up the const in the args. It really should be in there.
-        let opt_ct = self.args.get(p.index() as usize).map(|k| k.kind());
+        let opt_ct = self.args.get(p.index() as usize).map(|arg| arg.kind());
         let ct = match opt_ct {
             Some(ty::GenericArgKind::Const(ct)) => ct,
             Some(kind) => self.const_param_expected(p, source_ct, kind),
diff --git a/compiler/rustc_type_ir/src/canonical.rs b/compiler/rustc_type_ir/src/canonical.rs
index 67b67df4b28..2b1b0617cef 100644
--- a/compiler/rustc_type_ir/src/canonical.rs
+++ b/compiler/rustc_type_ir/src/canonical.rs
@@ -41,7 +41,7 @@ pub struct CanonicalQueryInput<I: Interner, V> {
 pub struct Canonical<I: Interner, V> {
     pub value: V,
     pub max_universe: UniverseIndex,
-    pub variables: I::CanonicalVars,
+    pub variables: I::CanonicalVarKinds,
 }
 
 impl<I: Interner, V> Canonical<I, V> {
@@ -89,63 +89,6 @@ impl<I: Interner, V: fmt::Display> fmt::Display for Canonical<I, V> {
 /// a copy of the canonical value in some other inference context,
 /// with fresh inference variables replacing the canonical values.
 #[derive_where(Clone, Copy, Hash, PartialEq, Eq, Debug; I: Interner)]
-#[derive(TypeVisitable_Generic, TypeFoldable_Generic)]
-#[cfg_attr(
-    feature = "nightly",
-    derive(Decodable_NoContext, Encodable_NoContext, HashStable_NoContext)
-)]
-pub struct CanonicalVarInfo<I: Interner> {
-    pub kind: CanonicalVarKind<I>,
-}
-
-impl<I: Interner> CanonicalVarInfo<I> {
-    pub fn universe(self) -> UniverseIndex {
-        self.kind.universe()
-    }
-
-    #[must_use]
-    pub fn with_updated_universe(self, ui: UniverseIndex) -> CanonicalVarInfo<I> {
-        CanonicalVarInfo { kind: self.kind.with_updated_universe(ui) }
-    }
-
-    pub fn is_existential(&self) -> bool {
-        match self.kind {
-            CanonicalVarKind::Ty(_) => true,
-            CanonicalVarKind::PlaceholderTy(_) => false,
-            CanonicalVarKind::Region(_) => true,
-            CanonicalVarKind::PlaceholderRegion(..) => false,
-            CanonicalVarKind::Const(_) => true,
-            CanonicalVarKind::PlaceholderConst(_) => false,
-        }
-    }
-
-    pub fn is_region(&self) -> bool {
-        match self.kind {
-            CanonicalVarKind::Region(_) | CanonicalVarKind::PlaceholderRegion(_) => true,
-            CanonicalVarKind::Ty(_)
-            | CanonicalVarKind::PlaceholderTy(_)
-            | CanonicalVarKind::Const(_)
-            | CanonicalVarKind::PlaceholderConst(_) => false,
-        }
-    }
-
-    pub fn expect_placeholder_index(self) -> usize {
-        match self.kind {
-            CanonicalVarKind::Ty(_) | CanonicalVarKind::Region(_) | CanonicalVarKind::Const(_) => {
-                panic!("expected placeholder: {self:?}")
-            }
-
-            CanonicalVarKind::PlaceholderRegion(placeholder) => placeholder.var().as_usize(),
-            CanonicalVarKind::PlaceholderTy(placeholder) => placeholder.var().as_usize(),
-            CanonicalVarKind::PlaceholderConst(placeholder) => placeholder.var().as_usize(),
-        }
-    }
-}
-
-/// Describes the "kind" of the canonical variable. This is a "kind"
-/// in the type-theory sense of the term -- i.e., a "meta" type system
-/// that analyzes type-like values.
-#[derive_where(Clone, Copy, Hash, PartialEq, Eq, Debug; I: Interner)]
 #[cfg_attr(
     feature = "nightly",
     derive(Decodable_NoContext, Encodable_NoContext, HashStable_NoContext)
@@ -214,6 +157,39 @@ impl<I: Interner> CanonicalVarKind<I> {
             }
         }
     }
+
+    pub fn is_existential(self) -> bool {
+        match self {
+            CanonicalVarKind::Ty(_) => true,
+            CanonicalVarKind::PlaceholderTy(_) => false,
+            CanonicalVarKind::Region(_) => true,
+            CanonicalVarKind::PlaceholderRegion(..) => false,
+            CanonicalVarKind::Const(_) => true,
+            CanonicalVarKind::PlaceholderConst(_) => false,
+        }
+    }
+
+    pub fn is_region(self) -> bool {
+        match self {
+            CanonicalVarKind::Region(_) | CanonicalVarKind::PlaceholderRegion(_) => true,
+            CanonicalVarKind::Ty(_)
+            | CanonicalVarKind::PlaceholderTy(_)
+            | CanonicalVarKind::Const(_)
+            | CanonicalVarKind::PlaceholderConst(_) => false,
+        }
+    }
+
+    pub fn expect_placeholder_index(self) -> usize {
+        match self {
+            CanonicalVarKind::Ty(_) | CanonicalVarKind::Region(_) | CanonicalVarKind::Const(_) => {
+                panic!("expected placeholder: {self:?}")
+            }
+
+            CanonicalVarKind::PlaceholderRegion(placeholder) => placeholder.var().as_usize(),
+            CanonicalVarKind::PlaceholderTy(placeholder) => placeholder.var().as_usize(),
+            CanonicalVarKind::PlaceholderConst(placeholder) => placeholder.var().as_usize(),
+        }
+    }
 }
 
 /// Rust actually has more than one category of type variables;
@@ -306,11 +282,11 @@ impl<I: Interner> CanonicalVarValues<I> {
 
     // Given a list of canonical variables, construct a set of values which are
     // the identity response.
-    pub fn make_identity(cx: I, infos: I::CanonicalVars) -> CanonicalVarValues<I> {
+    pub fn make_identity(cx: I, infos: I::CanonicalVarKinds) -> CanonicalVarValues<I> {
         CanonicalVarValues {
             var_values: cx.mk_args_from_iter(infos.iter().enumerate().map(
-                |(i, info)| -> I::GenericArg {
-                    match info.kind {
+                |(i, kind)| -> I::GenericArg {
+                    match kind {
                         CanonicalVarKind::Ty(_) | CanonicalVarKind::PlaceholderTy(_) => {
                             Ty::new_anon_bound(cx, ty::INNERMOST, ty::BoundVar::from_usize(i))
                                 .into()
diff --git a/compiler/rustc_type_ir/src/flags.rs b/compiler/rustc_type_ir/src/flags.rs
index 7ed0f92b639..37cc2baa402 100644
--- a/compiler/rustc_type_ir/src/flags.rs
+++ b/compiler/rustc_type_ir/src/flags.rs
@@ -479,8 +479,8 @@ impl<I: Interner> FlagComputation<I> {
     }
 
     fn add_args(&mut self, args: &[I::GenericArg]) {
-        for kind in args {
-            match kind.kind() {
+        for arg in args {
+            match arg.kind() {
                 ty::GenericArgKind::Type(ty) => self.add_ty(ty),
                 ty::GenericArgKind::Lifetime(lt) => self.add_region(lt),
                 ty::GenericArgKind::Const(ct) => self.add_const(ct),
diff --git a/compiler/rustc_type_ir/src/fold.rs b/compiler/rustc_type_ir/src/fold.rs
index ce1188070ca..a5eb8699e5f 100644
--- a/compiler/rustc_type_ir/src/fold.rs
+++ b/compiler/rustc_type_ir/src/fold.rs
@@ -152,6 +152,10 @@ pub trait TypeFolder<I: Interner>: Sized {
     fn fold_predicate(&mut self, p: I::Predicate) -> I::Predicate {
         p.super_fold_with(self)
     }
+
+    fn fold_clauses(&mut self, c: I::Clauses) -> I::Clauses {
+        c.super_fold_with(self)
+    }
 }
 
 /// This trait is implemented for every folding traversal. There is a fold
@@ -190,6 +194,10 @@ pub trait FallibleTypeFolder<I: Interner>: Sized {
     fn try_fold_predicate(&mut self, p: I::Predicate) -> Result<I::Predicate, Self::Error> {
         p.try_super_fold_with(self)
     }
+
+    fn try_fold_clauses(&mut self, c: I::Clauses) -> Result<I::Clauses, Self::Error> {
+        c.try_super_fold_with(self)
+    }
 }
 
 ///////////////////////////////////////////////////////////////////////////
diff --git a/compiler/rustc_type_ir/src/infer_ctxt.rs b/compiler/rustc_type_ir/src/infer_ctxt.rs
index c1490762117..2bc12d0a23b 100644
--- a/compiler/rustc_type_ir/src/infer_ctxt.rs
+++ b/compiler/rustc_type_ir/src/infer_ctxt.rs
@@ -1,11 +1,10 @@
-use std::fmt::Debug;
-
 use derive_where::derive_where;
 #[cfg(feature = "nightly")]
 use rustc_macros::{Decodable_NoContext, Encodable_NoContext, HashStable_NoContext};
 use rustc_type_ir_macros::{TypeFoldable_Generic, TypeVisitable_Generic};
 
 use crate::fold::TypeFoldable;
+use crate::inherent::*;
 use crate::relate::RelateResult;
 use crate::relate::combine::PredicateEmittingRelation;
 use crate::{self as ty, Interner};
@@ -168,6 +167,8 @@ pub trait InferCtxtLike: Sized {
         vid: ty::RegionVid,
     ) -> <Self::Interner as Interner>::Region;
 
+    fn is_changed_arg(&self, arg: <Self::Interner as Interner>::GenericArg) -> bool;
+
     fn next_region_infer(&self) -> <Self::Interner as Interner>::Region;
     fn next_ty_infer(&self) -> <Self::Interner as Interner>::Ty;
     fn next_const_infer(&self) -> <Self::Interner as Interner>::Const;
@@ -248,7 +249,7 @@ pub trait InferCtxtLike: Sized {
         span: <Self::Interner as Interner>::Span,
     );
 
-    type OpaqueTypeStorageEntries: Debug + Copy + Default;
+    type OpaqueTypeStorageEntries: OpaqueTypeStorageEntries;
     fn opaque_types_storage_num_entries(&self) -> Self::OpaqueTypeStorageEntries;
     fn clone_opaque_types_lookup_table(
         &self,
diff --git a/compiler/rustc_type_ir/src/inherent.rs b/compiler/rustc_type_ir/src/inherent.rs
index ee4a8096462..fa88bcb891a 100644
--- a/compiler/rustc_type_ir/src/inherent.rs
+++ b/compiler/rustc_type_ir/src/inherent.rs
@@ -297,6 +297,7 @@ pub trait GenericArg<I: Interner<GenericArg = Self>>:
     + From<I::Ty>
     + From<I::Region>
     + From<I::Const>
+    + From<I::Term>
 {
     fn as_term(&self) -> Option<I::Term> {
         match self.kind() {
@@ -510,6 +511,18 @@ pub trait Clause<I: Interner<Clause = Self>>:
     fn instantiate_supertrait(self, cx: I, trait_ref: ty::Binder<I, ty::TraitRef<I>>) -> Self;
 }
 
+pub trait Clauses<I: Interner<Clauses = Self>>:
+    Copy
+    + Debug
+    + Hash
+    + Eq
+    + TypeSuperVisitable<I>
+    + TypeSuperFoldable<I>
+    + Flags
+    + SliceLike<Item = I::Clause>
+{
+}
+
 /// Common capabilities of placeholder kinds
 pub trait PlaceholderLike: Copy + Debug + Hash + Eq {
     fn universe(self) -> ty::UniverseIndex;
@@ -596,6 +609,13 @@ pub trait Span<I: Interner>: Copy + Debug + Hash + Eq + TypeFoldable<I> {
     fn dummy() -> Self;
 }
 
+pub trait OpaqueTypeStorageEntries: Debug + Copy + Default {
+    /// Whether the number of opaques has changed in a way that necessitates
+    /// reevaluating a goal. For now, this is only when the number of non-duplicated
+    /// entries changed.
+    fn needs_reevaluation(self, canonicalized: usize) -> bool;
+}
+
 pub trait SliceLike: Sized + Copy {
     type Item: Copy;
     type IntoIter: Iterator<Item = Self::Item> + DoubleEndedIterator;
diff --git a/compiler/rustc_type_ir/src/interner.rs b/compiler/rustc_type_ir/src/interner.rs
index 0fd2d9f3ad3..05ca6f10323 100644
--- a/compiler/rustc_type_ir/src/interner.rs
+++ b/compiler/rustc_type_ir/src/interner.rs
@@ -4,7 +4,6 @@ use std::ops::Deref;
 
 use rustc_ast_ir::Movability;
 use rustc_index::bit_set::DenseBitSet;
-use smallvec::SmallVec;
 
 use crate::fold::TypeFoldable;
 use crate::inherent::*;
@@ -12,7 +11,7 @@ use crate::ir_print::IrPrint;
 use crate::lang_items::TraitSolverLangItem;
 use crate::relate::Relate;
 use crate::solve::{CanonicalInput, ExternalConstraintsData, PredefinedOpaquesData, QueryResult};
-use crate::visit::{Flags, TypeSuperVisitable, TypeVisitable};
+use crate::visit::{Flags, TypeVisitable};
 use crate::{self as ty, search_graph};
 
 #[cfg_attr(feature = "nightly", rustc_diagnostic_item = "type_ir_interner")]
@@ -64,13 +63,16 @@ pub trait Interner:
         + TypeVisitable<Self>
         + SliceLike<Item = Self::LocalDefId>;
 
-    type CanonicalVars: Copy
+    type CanonicalVarKinds: Copy
         + Debug
         + Hash
         + Eq
-        + SliceLike<Item = ty::CanonicalVarInfo<Self>>
+        + SliceLike<Item = ty::CanonicalVarKind<Self>>
         + Default;
-    fn mk_canonical_var_infos(self, infos: &[ty::CanonicalVarInfo<Self>]) -> Self::CanonicalVars;
+    fn mk_canonical_var_kinds(
+        self,
+        kinds: &[ty::CanonicalVarKind<Self>],
+    ) -> Self::CanonicalVarKinds;
 
     type ExternalConstraints: Copy
         + Debug
@@ -143,7 +145,7 @@ pub trait Interner:
     type ParamEnv: ParamEnv<Self>;
     type Predicate: Predicate<Self>;
     type Clause: Clause<Self>;
-    type Clauses: Copy + Debug + Hash + Eq + TypeSuperVisitable<Self> + Flags;
+    type Clauses: Clauses<Self>;
 
     fn with_global_cache<R>(self, f: impl FnOnce(&mut search_graph::GlobalCache<Self>) -> R) -> R;
 
@@ -210,7 +212,7 @@ pub trait Interner:
     fn coroutine_hidden_types(
         self,
         def_id: Self::DefId,
-    ) -> ty::EarlyBinder<Self, ty::Binder<Self, Self::Tys>>;
+    ) -> ty::EarlyBinder<Self, ty::Binder<Self, ty::CoroutineWitnessTypes<Self>>>;
 
     fn fn_sig(
         self,
@@ -379,28 +381,45 @@ impl<T, R> CollectAndApply<T, R> for T {
         F: FnOnce(&[T]) -> R,
     {
         // This code is hot enough that it's worth specializing for the most
-        // common length lists, to avoid the overhead of `SmallVec` creation.
-        // Lengths 0, 1, and 2 typically account for ~95% of cases. If
-        // `size_hint` is incorrect a panic will occur via an `unwrap` or an
-        // `assert`.
-        match iter.size_hint() {
-            (0, Some(0)) => {
-                assert!(iter.next().is_none());
-                f(&[])
-            }
-            (1, Some(1)) => {
-                let t0 = iter.next().unwrap();
-                assert!(iter.next().is_none());
-                f(&[t0])
-            }
-            (2, Some(2)) => {
-                let t0 = iter.next().unwrap();
-                let t1 = iter.next().unwrap();
-                assert!(iter.next().is_none());
-                f(&[t0, t1])
-            }
-            _ => f(&iter.collect::<SmallVec<[_; 8]>>()),
-        }
+        // common length lists, to avoid the overhead of `Vec` creation.
+
+        let Some(t0) = iter.next() else {
+            return f(&[]);
+        };
+
+        let Some(t1) = iter.next() else {
+            return f(&[t0]);
+        };
+
+        let Some(t2) = iter.next() else {
+            return f(&[t0, t1]);
+        };
+
+        let Some(t3) = iter.next() else {
+            return f(&[t0, t1, t2]);
+        };
+
+        let Some(t4) = iter.next() else {
+            return f(&[t0, t1, t2, t3]);
+        };
+
+        let Some(t5) = iter.next() else {
+            return f(&[t0, t1, t2, t3, t4]);
+        };
+
+        let Some(t6) = iter.next() else {
+            return f(&[t0, t1, t2, t3, t4, t5]);
+        };
+
+        let Some(t7) = iter.next() else {
+            return f(&[t0, t1, t2, t3, t4, t5, t6]);
+        };
+
+        let Some(t8) = iter.next() else {
+            return f(&[t0, t1, t2, t3, t4, t5, t6, t7]);
+        };
+
+        f(&[t0, t1, t2, t3, t4, t5, t6, t7, t8].into_iter().chain(iter).collect::<Vec<_>>())
     }
 }
 
@@ -416,29 +435,57 @@ impl<T, R, E> CollectAndApply<T, R> for Result<T, E> {
         F: FnOnce(&[T]) -> R,
     {
         // This code is hot enough that it's worth specializing for the most
-        // common length lists, to avoid the overhead of `SmallVec` creation.
-        // Lengths 0, 1, and 2 typically account for ~95% of cases. If
-        // `size_hint` is incorrect a panic will occur via an `unwrap` or an
-        // `assert`, unless a failure happens first, in which case the result
-        // will be an error anyway.
-        Ok(match iter.size_hint() {
-            (0, Some(0)) => {
-                assert!(iter.next().is_none());
-                f(&[])
-            }
-            (1, Some(1)) => {
-                let t0 = iter.next().unwrap()?;
-                assert!(iter.next().is_none());
-                f(&[t0])
-            }
-            (2, Some(2)) => {
-                let t0 = iter.next().unwrap()?;
-                let t1 = iter.next().unwrap()?;
-                assert!(iter.next().is_none());
-                f(&[t0, t1])
-            }
-            _ => f(&iter.collect::<Result<SmallVec<[_; 8]>, _>>()?),
-        })
+        // common length lists, to avoid the overhead of `Vec` creation.
+
+        let Some(t0) = iter.next() else {
+            return Ok(f(&[]));
+        };
+        let t0 = t0?;
+
+        let Some(t1) = iter.next() else {
+            return Ok(f(&[t0]));
+        };
+        let t1 = t1?;
+
+        let Some(t2) = iter.next() else {
+            return Ok(f(&[t0, t1]));
+        };
+        let t2 = t2?;
+
+        let Some(t3) = iter.next() else {
+            return Ok(f(&[t0, t1, t2]));
+        };
+        let t3 = t3?;
+
+        let Some(t4) = iter.next() else {
+            return Ok(f(&[t0, t1, t2, t3]));
+        };
+        let t4 = t4?;
+
+        let Some(t5) = iter.next() else {
+            return Ok(f(&[t0, t1, t2, t3, t4]));
+        };
+        let t5 = t5?;
+
+        let Some(t6) = iter.next() else {
+            return Ok(f(&[t0, t1, t2, t3, t4, t5]));
+        };
+        let t6 = t6?;
+
+        let Some(t7) = iter.next() else {
+            return Ok(f(&[t0, t1, t2, t3, t4, t5, t6]));
+        };
+        let t7 = t7?;
+
+        let Some(t8) = iter.next() else {
+            return Ok(f(&[t0, t1, t2, t3, t4, t5, t6, t7]));
+        };
+        let t8 = t8?;
+
+        Ok(f(&[Ok(t0), Ok(t1), Ok(t2), Ok(t3), Ok(t4), Ok(t5), Ok(t6), Ok(t7), Ok(t8)]
+            .into_iter()
+            .chain(iter)
+            .collect::<Result<Vec<_>, _>>()?))
     }
 }
 
diff --git a/compiler/rustc_type_ir/src/region_kind.rs b/compiler/rustc_type_ir/src/region_kind.rs
index eae3213ead0..1b5d04e6025 100644
--- a/compiler/rustc_type_ir/src/region_kind.rs
+++ b/compiler/rustc_type_ir/src/region_kind.rs
@@ -193,7 +193,7 @@ impl<I: Interner> fmt::Debug for RegionKind<I> {
 
             ReVar(vid) => write!(f, "{vid:?}"),
 
-            RePlaceholder(placeholder) => write!(f, "{placeholder:?}"),
+            RePlaceholder(placeholder) => write!(f, "'{placeholder:?}"),
 
             // Use `'{erased}` as the output instead of `'erased` so that its more obviously distinct from
             // a `ReEarlyParam` named `'erased`. Technically that would print as `'erased/#IDX` so this is
diff --git a/compiler/rustc_type_ir/src/ty_kind.rs b/compiler/rustc_type_ir/src/ty_kind.rs
index cf2e4284d10..0cd98b5aa53 100644
--- a/compiler/rustc_type_ir/src/ty_kind.rs
+++ b/compiler/rustc_type_ir/src/ty_kind.rs
@@ -1163,3 +1163,13 @@ pub struct FnHeader<I: Interner> {
     pub safety: I::Safety,
     pub abi: I::Abi,
 }
+
+#[derive_where(Clone, Copy, Debug, PartialEq, Eq, Hash; I: Interner)]
+#[cfg_attr(
+    feature = "nightly",
+    derive(Encodable_NoContext, Decodable_NoContext, HashStable_NoContext)
+)]
+#[derive(TypeVisitable_Generic, TypeFoldable_Generic, Lift_Generic)]
+pub struct CoroutineWitnessTypes<I: Interner> {
+    pub types: I::Tys,
+}
diff --git a/compiler/rustc_type_ir/src/visit.rs b/compiler/rustc_type_ir/src/visit.rs
index ccb84e25911..fc3864dd5ae 100644
--- a/compiler/rustc_type_ir/src/visit.rs
+++ b/compiler/rustc_type_ir/src/visit.rs
@@ -120,8 +120,8 @@ pub trait TypeVisitor<I: Interner>: Sized {
         p.super_visit_with(self)
     }
 
-    fn visit_clauses(&mut self, p: I::Clauses) -> Self::Result {
-        p.super_visit_with(self)
+    fn visit_clauses(&mut self, c: I::Clauses) -> Self::Result {
+        c.super_visit_with(self)
     }
 
     fn visit_error(&mut self, _guar: I::ErrorGuaranteed) -> Self::Result {