Auto merge of #3876 - RalfJung:rustup, r=RalfJung

Rustup

232 files changed, 3271 insertions, 2209 deletions

diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs
index 8a66894a356..7c8af6c4a55 100644
--- a/compiler/rustc_ast/src/mut_visit.rs
+++ b/compiler/rustc_ast/src/mut_visit.rs
@@ -752,7 +752,7 @@ fn visit_lazy_tts<T: MutVisitor>(vis: &mut T, lazy_tts: &mut Option<LazyAttrToke
 pub fn visit_token<T: MutVisitor>(vis: &mut T, t: &mut Token) {
     let Token { kind, span } = t;
     match kind {
-        token::Ident(name, _ /*raw*/) | token::Lifetime(name) => {
+        token::Ident(name, _is_raw) | token::Lifetime(name, _is_raw) => {
             let mut ident = Ident::new(*name, *span);
             vis.visit_ident(&mut ident);
             *name = ident.name;
@@ -762,7 +762,7 @@ pub fn visit_token<T: MutVisitor>(vis: &mut T, t: &mut Token) {
         token::NtIdent(ident, _is_raw) => {
             vis.visit_ident(ident);
         }
-        token::NtLifetime(ident) => {
+        token::NtLifetime(ident, _is_raw) => {
             vis.visit_ident(ident);
         }
         token::Interpolated(nt) => {
diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs
index f1dddb3acac..a0082a41713 100644
--- a/compiler/rustc_ast/src/token.rs
+++ b/compiler/rustc_ast/src/token.rs
@@ -331,11 +331,11 @@ pub enum TokenKind {
     /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
     /// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to
     /// treat regular and interpolated lifetime identifiers in the same way.
-    Lifetime(Symbol),
+    Lifetime(Symbol, IdentIsRaw),
     /// This identifier (and its span) is the lifetime passed to the
     /// declarative macro. The span in the surrounding `Token` is the span of
     /// the `lifetime` metavariable in the macro's RHS.
-    NtLifetime(Ident),
+    NtLifetime(Ident, IdentIsRaw),
 
     /// An embedded AST node, as produced by a macro. This only exists for
     /// historical reasons. We'd like to get rid of it, for multiple reasons.
@@ -458,7 +458,7 @@ impl Token {
     /// if they keep spans or perform edition checks.
     pub fn uninterpolated_span(&self) -> Span {
         match self.kind {
-            NtIdent(ident, _) | NtLifetime(ident) => ident.span,
+            NtIdent(ident, _) | NtLifetime(ident, _) => ident.span,
             Interpolated(ref nt) => nt.use_span(),
             _ => self.span,
         }
@@ -661,7 +661,9 @@ impl Token {
     pub fn uninterpolate(&self) -> Cow<'_, Token> {
         match self.kind {
             NtIdent(ident, is_raw) => Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span)),
-            NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
+            NtLifetime(ident, is_raw) => {
+                Cow::Owned(Token::new(Lifetime(ident.name, is_raw), ident.span))
+            }
             _ => Cow::Borrowed(self),
         }
     }
@@ -679,11 +681,11 @@ impl Token {
 
     /// Returns a lifetime identifier if this token is a lifetime.
     #[inline]
-    pub fn lifetime(&self) -> Option<Ident> {
+    pub fn lifetime(&self) -> Option<(Ident, IdentIsRaw)> {
         // We avoid using `Token::uninterpolate` here because it's slow.
         match self.kind {
-            Lifetime(name) => Some(Ident::new(name, self.span)),
-            NtLifetime(ident) => Some(ident),
+            Lifetime(name, is_raw) => Some((Ident::new(name, self.span), is_raw)),
+            NtLifetime(ident, is_raw) => Some((ident, is_raw)),
             _ => None,
         }
     }
@@ -865,7 +867,7 @@ impl Token {
                 _ => return None,
             },
             SingleQuote => match joint.kind {
-                Ident(name, IdentIsRaw::No) => Lifetime(Symbol::intern(&format!("'{name}"))),
+                Ident(name, is_raw) => Lifetime(Symbol::intern(&format!("'{name}")), is_raw),
                 _ => return None,
             },
 
diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs
index 057b4455dca..fc1dd2caf68 100644
--- a/compiler/rustc_ast/src/tokenstream.rs
+++ b/compiler/rustc_ast/src/tokenstream.rs
@@ -482,11 +482,11 @@ impl TokenStream {
             token::NtIdent(ident, is_raw) => {
                 TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing)
             }
-            token::NtLifetime(ident) => TokenTree::Delimited(
+            token::NtLifetime(ident, is_raw) => TokenTree::Delimited(
                 DelimSpan::from_single(token.span),
                 DelimSpacing::new(Spacing::JointHidden, spacing),
                 Delimiter::Invisible,
-                TokenStream::token_alone(token::Lifetime(ident.name), ident.span),
+                TokenStream::token_alone(token::Lifetime(ident.name, is_raw), ident.span),
             ),
             token::Interpolated(ref nt) => TokenTree::Delimited(
                 DelimSpan::from_single(token.span),
diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs
index c7ff39d23ed..3b1449d9a91 100644
--- a/compiler/rustc_ast_pretty/src/pprust/state.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state.rs
@@ -11,7 +11,9 @@ use std::borrow::Cow;
 use ast::TraitBoundModifiers;
 use rustc_ast::attr::AttrIdGenerator;
 use rustc_ast::ptr::P;
-use rustc_ast::token::{self, BinOpToken, CommentKind, Delimiter, Nonterminal, Token, TokenKind};
+use rustc_ast::token::{
+    self, BinOpToken, CommentKind, Delimiter, IdentIsRaw, Nonterminal, Token, TokenKind,
+};
 use rustc_ast::tokenstream::{Spacing, TokenStream, TokenTree};
 use rustc_ast::util::classify;
 use rustc_ast::util::comments::{Comment, CommentStyle};
@@ -947,8 +949,13 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
             token::NtIdent(ident, is_raw) => {
                 IdentPrinter::for_ast_ident(ident, is_raw.into()).to_string().into()
             }
-            token::Lifetime(name) => name.to_string().into(),
-            token::NtLifetime(ident) => ident.name.to_string().into(),
+
+            token::Lifetime(name, IdentIsRaw::No)
+            | token::NtLifetime(Ident { name, .. }, IdentIsRaw::No) => name.to_string().into(),
+            token::Lifetime(name, IdentIsRaw::Yes)
+            | token::NtLifetime(Ident { name, .. }, IdentIsRaw::Yes) => {
+                format!("'r#{}", &name.as_str()[1..]).into()
+            }
 
             /* Other */
             token::DocComment(comment_kind, attr_style, data) => {
diff --git a/compiler/rustc_borrowck/src/borrowck_errors.rs b/compiler/rustc_borrowck/src/borrowck_errors.rs
index 2c672dbf8c4..3a7e407f3e7 100644
--- a/compiler/rustc_borrowck/src/borrowck_errors.rs
+++ b/compiler/rustc_borrowck/src/borrowck_errors.rs
@@ -8,7 +8,7 @@ use rustc_middle::span_bug;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_span::Span;
 
-impl<'infcx, 'tcx> crate::MirBorrowckCtxt<'_, '_, 'infcx, 'tcx> {
+impl<'infcx, 'tcx> crate::MirBorrowckCtxt<'_, 'infcx, 'tcx> {
     pub(crate) fn dcx(&self) -> DiagCtxtHandle<'infcx> {
         self.infcx.dcx()
     }
diff --git a/compiler/rustc_borrowck/src/dataflow.rs b/compiler/rustc_borrowck/src/dataflow.rs
index 9bb6109e66d..39994ad784a 100644
--- a/compiler/rustc_borrowck/src/dataflow.rs
+++ b/compiler/rustc_borrowck/src/dataflow.rs
@@ -15,24 +15,24 @@ use tracing::debug;
 use crate::{places_conflict, BorrowSet, PlaceConflictBias, PlaceExt, RegionInferenceContext};
 
 /// The results of the dataflow analyses used by the borrow checker.
-pub(crate) struct BorrowckResults<'a, 'mir, 'tcx> {
-    pub(crate) borrows: Results<'tcx, Borrows<'a, 'mir, 'tcx>>,
-    pub(crate) uninits: Results<'tcx, MaybeUninitializedPlaces<'a, 'mir, 'tcx>>,
-    pub(crate) ever_inits: Results<'tcx, EverInitializedPlaces<'a, 'mir, 'tcx>>,
+pub(crate) struct BorrowckResults<'a, 'tcx> {
+    pub(crate) borrows: Results<'tcx, Borrows<'a, 'tcx>>,
+    pub(crate) uninits: Results<'tcx, MaybeUninitializedPlaces<'a, 'tcx>>,
+    pub(crate) ever_inits: Results<'tcx, EverInitializedPlaces<'a, 'tcx>>,
 }
 
 /// The transient state of the dataflow analyses used by the borrow checker.
 #[derive(Debug)]
-pub(crate) struct BorrowckFlowState<'a, 'mir, 'tcx> {
-    pub(crate) borrows: <Borrows<'a, 'mir, 'tcx> as AnalysisDomain<'tcx>>::Domain,
-    pub(crate) uninits: <MaybeUninitializedPlaces<'a, 'mir, 'tcx> as AnalysisDomain<'tcx>>::Domain,
-    pub(crate) ever_inits: <EverInitializedPlaces<'a, 'mir, 'tcx> as AnalysisDomain<'tcx>>::Domain,
+pub(crate) struct BorrowckFlowState<'a, 'tcx> {
+    pub(crate) borrows: <Borrows<'a, 'tcx> as AnalysisDomain<'tcx>>::Domain,
+    pub(crate) uninits: <MaybeUninitializedPlaces<'a, 'tcx> as AnalysisDomain<'tcx>>::Domain,
+    pub(crate) ever_inits: <EverInitializedPlaces<'a, 'tcx> as AnalysisDomain<'tcx>>::Domain,
 }
 
-impl<'a, 'mir, 'tcx> ResultsVisitable<'tcx> for BorrowckResults<'a, 'mir, 'tcx> {
+impl<'a, 'tcx> ResultsVisitable<'tcx> for BorrowckResults<'a, 'tcx> {
     // All three analyses are forward, but we have to use just one here.
-    type Direction = <Borrows<'a, 'mir, 'tcx> as AnalysisDomain<'tcx>>::Direction;
-    type FlowState = BorrowckFlowState<'a, 'mir, 'tcx>;
+    type Direction = <Borrows<'a, 'tcx> as AnalysisDomain<'tcx>>::Direction;
+    type FlowState = BorrowckFlowState<'a, 'tcx>;
 
     fn new_flow_state(&self, body: &mir::Body<'tcx>) -> Self::FlowState {
         BorrowckFlowState {
@@ -106,10 +106,9 @@ rustc_index::newtype_index! {
 /// `BorrowIndex`, and maps each such index to a `BorrowData`
 /// describing the borrow. These indexes are used for representing the
 /// borrows in compact bitvectors.
-pub struct Borrows<'a, 'mir, 'tcx> {
+pub struct Borrows<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
-    body: &'mir Body<'tcx>,
-
+    body: &'a Body<'tcx>,
     borrow_set: &'a BorrowSet<'tcx>,
     borrows_out_of_scope_at_location: FxIndexMap<Location, Vec<BorrowIndex>>,
 }
@@ -389,10 +388,10 @@ impl<'tcx> PoloniusOutOfScopePrecomputer<'_, 'tcx> {
     }
 }
 
-impl<'a, 'mir, 'tcx> Borrows<'a, 'mir, 'tcx> {
+impl<'a, 'tcx> Borrows<'a, 'tcx> {
     pub fn new(
         tcx: TyCtxt<'tcx>,
-        body: &'mir Body<'tcx>,
+        body: &'a Body<'tcx>,
         regioncx: &RegionInferenceContext<'tcx>,
         borrow_set: &'a BorrowSet<'tcx>,
     ) -> Self {
@@ -494,7 +493,7 @@ impl<'a, 'mir, 'tcx> Borrows<'a, 'mir, 'tcx> {
     }
 }
 
-impl<'tcx> rustc_mir_dataflow::AnalysisDomain<'tcx> for Borrows<'_, '_, 'tcx> {
+impl<'tcx> rustc_mir_dataflow::AnalysisDomain<'tcx> for Borrows<'_, 'tcx> {
     type Domain = BitSet<BorrowIndex>;
 
     const NAME: &'static str = "borrows";
@@ -517,7 +516,7 @@ impl<'tcx> rustc_mir_dataflow::AnalysisDomain<'tcx> for Borrows<'_, '_, 'tcx> {
 ///   region stops containing the CFG points reachable from the issuing location.
 /// - we also kill loans of conflicting places when overwriting a shared path: e.g. borrows of
 ///   `a.b.c` when `a` is overwritten.
-impl<'tcx> rustc_mir_dataflow::GenKillAnalysis<'tcx> for Borrows<'_, '_, 'tcx> {
+impl<'tcx> rustc_mir_dataflow::GenKillAnalysis<'tcx> for Borrows<'_, 'tcx> {
     type Idx = BorrowIndex;
 
     fn domain_size(&self, _: &mir::Body<'tcx>) -> usize {
@@ -617,8 +616,8 @@ impl<'tcx> rustc_mir_dataflow::GenKillAnalysis<'tcx> for Borrows<'_, '_, 'tcx> {
     }
 }
 
-impl DebugWithContext<Borrows<'_, '_, '_>> for BorrowIndex {
-    fn fmt_with(&self, ctxt: &Borrows<'_, '_, '_>, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+impl DebugWithContext<Borrows<'_, '_>> for BorrowIndex {
+    fn fmt_with(&self, ctxt: &Borrows<'_, '_>, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "{:?}", ctxt.location(*self))
     }
 }
diff --git a/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs b/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs
index 8a4e89d47bd..ee28e556cbc 100644
--- a/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs
@@ -52,7 +52,7 @@ impl<'tcx> UniverseInfo<'tcx> {
 
     pub(crate) fn report_error(
         &self,
-        mbcx: &mut MirBorrowckCtxt<'_, '_, '_, 'tcx>,
+        mbcx: &mut MirBorrowckCtxt<'_, '_, 'tcx>,
         placeholder: ty::PlaceholderRegion,
         error_element: RegionElement,
         cause: ObligationCause<'tcx>,
@@ -151,7 +151,7 @@ trait TypeOpInfo<'tcx> {
 
     fn nice_error<'infcx>(
         &self,
-        mbcx: &mut MirBorrowckCtxt<'_, '_, 'infcx, 'tcx>,
+        mbcx: &mut MirBorrowckCtxt<'_, 'infcx, 'tcx>,
         cause: ObligationCause<'tcx>,
         placeholder_region: ty::Region<'tcx>,
         error_region: Option<ty::Region<'tcx>>,
@@ -160,7 +160,7 @@ trait TypeOpInfo<'tcx> {
     #[instrument(level = "debug", skip(self, mbcx))]
     fn report_error(
         &self,
-        mbcx: &mut MirBorrowckCtxt<'_, '_, '_, 'tcx>,
+        mbcx: &mut MirBorrowckCtxt<'_, '_, 'tcx>,
         placeholder: ty::PlaceholderRegion,
         error_element: RegionElement,
         cause: ObligationCause<'tcx>,
@@ -233,7 +233,7 @@ impl<'tcx> TypeOpInfo<'tcx> for PredicateQuery<'tcx> {
 
     fn nice_error<'infcx>(
         &self,
-        mbcx: &mut MirBorrowckCtxt<'_, '_, 'infcx, 'tcx>,
+        mbcx: &mut MirBorrowckCtxt<'_, 'infcx, 'tcx>,
         cause: ObligationCause<'tcx>,
         placeholder_region: ty::Region<'tcx>,
         error_region: Option<ty::Region<'tcx>>,
@@ -277,7 +277,7 @@ where
 
     fn nice_error<'infcx>(
         &self,
-        mbcx: &mut MirBorrowckCtxt<'_, '_, 'infcx, 'tcx>,
+        mbcx: &mut MirBorrowckCtxt<'_, 'infcx, 'tcx>,
         cause: ObligationCause<'tcx>,
         placeholder_region: ty::Region<'tcx>,
         error_region: Option<ty::Region<'tcx>>,
@@ -324,7 +324,7 @@ impl<'tcx> TypeOpInfo<'tcx> for AscribeUserTypeQuery<'tcx> {
 
     fn nice_error<'infcx>(
         &self,
-        mbcx: &mut MirBorrowckCtxt<'_, '_, 'infcx, 'tcx>,
+        mbcx: &mut MirBorrowckCtxt<'_, 'infcx, 'tcx>,
         cause: ObligationCause<'tcx>,
         placeholder_region: ty::Region<'tcx>,
         error_region: Option<ty::Region<'tcx>>,
@@ -357,7 +357,7 @@ impl<'tcx> TypeOpInfo<'tcx> for crate::type_check::InstantiateOpaqueType<'tcx> {
 
     fn nice_error<'infcx>(
         &self,
-        mbcx: &mut MirBorrowckCtxt<'_, '_, 'infcx, 'tcx>,
+        mbcx: &mut MirBorrowckCtxt<'_, 'infcx, 'tcx>,
         _cause: ObligationCause<'tcx>,
         placeholder_region: ty::Region<'tcx>,
         error_region: Option<ty::Region<'tcx>>,
diff --git a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
index c817b6fac71..a47518fca3f 100644
--- a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
@@ -69,7 +69,7 @@ enum StorageDeadOrDrop<'tcx> {
     Destructor(Ty<'tcx>),
 }
 
-impl<'infcx, 'tcx> MirBorrowckCtxt<'_, '_, 'infcx, 'tcx> {
+impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
     pub(crate) fn report_use_of_moved_or_uninitialized(
         &mut self,
         location: Location,
@@ -4358,11 +4358,7 @@ enum AnnotatedBorrowFnSignature<'tcx> {
 impl<'tcx> AnnotatedBorrowFnSignature<'tcx> {
     /// Annotate the provided diagnostic with information about borrow from the fn signature that
     /// helps explain.
-    pub(crate) fn emit(
-        &self,
-        cx: &MirBorrowckCtxt<'_, '_, '_, 'tcx>,
-        diag: &mut Diag<'_>,
-    ) -> String {
+    pub(crate) fn emit(&self, cx: &MirBorrowckCtxt<'_, '_, 'tcx>, diag: &mut Diag<'_>) -> String {
         match self {
             &AnnotatedBorrowFnSignature::Closure { argument_ty, argument_span } => {
                 diag.span_label(
diff --git a/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs b/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
index 91b02a36d00..419e72df00b 100644
--- a/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
@@ -390,7 +390,7 @@ impl<'tcx> BorrowExplanation<'tcx> {
     }
 }
 
-impl<'tcx> MirBorrowckCtxt<'_, '_, '_, 'tcx> {
+impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
     fn free_region_constraint_info(
         &self,
         borrow_region: RegionVid,
@@ -662,9 +662,10 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, '_, 'tcx> {
                                                 // `&dyn Trait`
                                                 ty::Ref(_, ty, _) if ty.is_trait() => true,
                                                 // `Box<dyn Trait>`
-                                                _ if ty.is_box() && ty.boxed_ty().is_trait() => {
+                                                _ if ty.boxed_ty().is_some_and(Ty::is_trait) => {
                                                     true
                                                 }
+
                                                 // `dyn Trait`
                                                 _ if ty.is_trait() => true,
                                                 // Anything else.
diff --git a/compiler/rustc_borrowck/src/diagnostics/mod.rs b/compiler/rustc_borrowck/src/diagnostics/mod.rs
index 5ab66963409..e52ddfa3eb5 100644
--- a/compiler/rustc_borrowck/src/diagnostics/mod.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/mod.rs
@@ -68,7 +68,7 @@ pub(super) struct DescribePlaceOpt {
 
 pub(super) struct IncludingTupleField(pub(super) bool);
 
-impl<'infcx, 'tcx> MirBorrowckCtxt<'_, '_, 'infcx, 'tcx> {
+impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
     /// Adds a suggestion when a closure is invoked twice with a moved variable or when a closure
     /// is moved after being invoked.
     ///
@@ -345,9 +345,9 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, '_, 'infcx, 'tcx> {
         variant_index: Option<VariantIdx>,
         including_tuple_field: IncludingTupleField,
     ) -> Option<String> {
-        if ty.is_box() {
+        if let Some(boxed_ty) = ty.boxed_ty() {
             // If the type is a box, the field is described from the boxed type
-            self.describe_field_from_ty(ty.boxed_ty(), field, variant_index, including_tuple_field)
+            self.describe_field_from_ty(boxed_ty, field, variant_index, including_tuple_field)
         } else {
             match *ty.kind() {
                 ty::Adt(def, _) => {
@@ -772,7 +772,7 @@ struct CapturedMessageOpt {
     maybe_reinitialized_locations_is_empty: bool,
 }
 
-impl<'infcx, 'tcx> MirBorrowckCtxt<'_, '_, 'infcx, 'tcx> {
+impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
     /// Finds the spans associated to a move or copy of move_place at location.
     pub(super) fn move_spans(
         &self,
diff --git a/compiler/rustc_borrowck/src/diagnostics/move_errors.rs b/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
index 3dab027bff0..5a9eba34d07 100644
--- a/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
@@ -93,7 +93,7 @@ enum GroupedMoveError<'tcx> {
     },
 }
 
-impl<'infcx, 'tcx> MirBorrowckCtxt<'_, '_, 'infcx, 'tcx> {
+impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
     pub(crate) fn report_move_errors(&mut self) {
         let grouped_errors = self.group_move_errors();
         for error in grouped_errors {
diff --git a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
index 337125f5ecc..6e3fac1e680 100644
--- a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
@@ -32,7 +32,7 @@ pub(crate) enum AccessKind {
     Mutate,
 }
 
-impl<'infcx, 'tcx> MirBorrowckCtxt<'_, '_, 'infcx, 'tcx> {
+impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
     pub(crate) fn report_mutability_error(
         &mut self,
         access_place: Place<'tcx>,
diff --git a/compiler/rustc_borrowck/src/diagnostics/outlives_suggestion.rs b/compiler/rustc_borrowck/src/diagnostics/outlives_suggestion.rs
index b205dc9ff49..bd0cf3578c2 100644
--- a/compiler/rustc_borrowck/src/diagnostics/outlives_suggestion.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/outlives_suggestion.rs
@@ -76,7 +76,7 @@ impl OutlivesSuggestionBuilder {
     /// Returns a name for the region if it is suggestable. See `region_name_is_suggestable`.
     fn region_vid_to_name(
         &self,
-        mbcx: &MirBorrowckCtxt<'_, '_, '_, '_>,
+        mbcx: &MirBorrowckCtxt<'_, '_, '_>,
         region: RegionVid,
     ) -> Option<RegionName> {
         mbcx.give_region_a_name(region).filter(Self::region_name_is_suggestable)
@@ -85,7 +85,7 @@ impl OutlivesSuggestionBuilder {
     /// Compiles a list of all suggestions to be printed in the final big suggestion.
     fn compile_all_suggestions(
         &self,
-        mbcx: &MirBorrowckCtxt<'_, '_, '_, '_>,
+        mbcx: &MirBorrowckCtxt<'_, '_, '_>,
     ) -> SmallVec<[SuggestedConstraint; 2]> {
         let mut suggested = SmallVec::new();
 
@@ -161,7 +161,7 @@ impl OutlivesSuggestionBuilder {
     /// Emit an intermediate note on the given `Diag` if the involved regions are suggestable.
     pub(crate) fn intermediate_suggestion(
         &mut self,
-        mbcx: &MirBorrowckCtxt<'_, '_, '_, '_>,
+        mbcx: &MirBorrowckCtxt<'_, '_, '_>,
         errci: &ErrorConstraintInfo<'_>,
         diag: &mut Diag<'_>,
     ) {
@@ -180,7 +180,7 @@ impl OutlivesSuggestionBuilder {
 
     /// If there is a suggestion to emit, add a diagnostic to the buffer. This is the final
     /// suggestion including all collected constraints.
-    pub(crate) fn add_suggestion(&self, mbcx: &mut MirBorrowckCtxt<'_, '_, '_, '_>) {
+    pub(crate) fn add_suggestion(&self, mbcx: &mut MirBorrowckCtxt<'_, '_, '_>) {
         // No constraints to add? Done.
         if self.constraints_to_add.is_empty() {
             debug!("No constraints to suggest.");
diff --git a/compiler/rustc_borrowck/src/diagnostics/region_errors.rs b/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
index d49d36dedaf..57c3a0843a6 100644
--- a/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
@@ -156,7 +156,7 @@ pub(crate) struct ErrorConstraintInfo<'tcx> {
     pub(super) span: Span,
 }
 
-impl<'infcx, 'tcx> MirBorrowckCtxt<'_, '_, 'infcx, 'tcx> {
+impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
     /// Converts a region inference variable into a `ty::Region` that
     /// we can use for error reporting. If `r` is universally bound,
     /// then we use the name that we have on record for it. If `r` is
diff --git a/compiler/rustc_borrowck/src/diagnostics/region_name.rs b/compiler/rustc_borrowck/src/diagnostics/region_name.rs
index cb05812ec7b..58fcda2571c 100644
--- a/compiler/rustc_borrowck/src/diagnostics/region_name.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/region_name.rs
@@ -200,7 +200,7 @@ impl rustc_errors::IntoDiagArg for RegionName {
     }
 }
 
-impl<'tcx> MirBorrowckCtxt<'_, '_, '_, 'tcx> {
+impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
     pub(crate) fn mir_def_id(&self) -> hir::def_id::LocalDefId {
         self.body.source.def_id().expect_local()
     }
diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs
index ffb350b1d1f..d5f297a5913 100644
--- a/compiler/rustc_borrowck/src/lib.rs
+++ b/compiler/rustc_borrowck/src/lib.rs
@@ -304,11 +304,11 @@ fn do_mir_borrowck<'tcx>(
         promoted_mbcx.report_move_errors();
         diags = promoted_mbcx.diags;
 
-        struct MoveVisitor<'a, 'b, 'mir, 'infcx, 'tcx> {
-            ctxt: &'a mut MirBorrowckCtxt<'b, 'mir, 'infcx, 'tcx>,
+        struct MoveVisitor<'a, 'b, 'infcx, 'tcx> {
+            ctxt: &'a mut MirBorrowckCtxt<'b, 'infcx, 'tcx>,
         }
 
-        impl<'tcx> Visitor<'tcx> for MoveVisitor<'_, '_, '_, '_, 'tcx> {
+        impl<'tcx> Visitor<'tcx> for MoveVisitor<'_, '_, '_, 'tcx> {
             fn visit_operand(&mut self, operand: &Operand<'tcx>, location: Location) {
                 if let Operand::Move(place) = operand {
                     self.ctxt.check_movable_place(location, *place);
@@ -522,10 +522,10 @@ impl<'tcx> Deref for BorrowckInferCtxt<'tcx> {
     }
 }
 
-struct MirBorrowckCtxt<'a, 'mir, 'infcx, 'tcx> {
+struct MirBorrowckCtxt<'a, 'infcx, 'tcx> {
     infcx: &'infcx BorrowckInferCtxt<'tcx>,
     param_env: ParamEnv<'tcx>,
-    body: &'mir Body<'tcx>,
+    body: &'a Body<'tcx>,
     move_data: &'a MoveData<'tcx>,
 
     /// Map from MIR `Location` to `LocationIndex`; created
@@ -599,16 +599,16 @@ struct MirBorrowckCtxt<'a, 'mir, 'infcx, 'tcx> {
 // 2. loans made in overlapping scopes do not conflict
 // 3. assignments do not affect things loaned out as immutable
 // 4. moves do not affect things loaned out in any way
-impl<'a, 'mir, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'mir, 'tcx, R>
-    for MirBorrowckCtxt<'a, 'mir, '_, 'tcx>
+impl<'a, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'a, 'tcx, R>
+    for MirBorrowckCtxt<'a, '_, 'tcx>
 {
-    type FlowState = Flows<'a, 'mir, 'tcx>;
+    type FlowState = Flows<'a, 'tcx>;
 
     fn visit_statement_before_primary_effect(
         &mut self,
         _results: &mut R,
-        flow_state: &Flows<'_, 'mir, 'tcx>,
-        stmt: &'mir Statement<'tcx>,
+        flow_state: &Flows<'a, 'tcx>,
+        stmt: &'a Statement<'tcx>,
         location: Location,
     ) {
         debug!("MirBorrowckCtxt::process_statement({:?}, {:?}): {:?}", location, stmt, flow_state);
@@ -677,8 +677,8 @@ impl<'a, 'mir, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'mir, 'tcx, R>
     fn visit_terminator_before_primary_effect(
         &mut self,
         _results: &mut R,
-        flow_state: &Flows<'_, 'mir, 'tcx>,
-        term: &'mir Terminator<'tcx>,
+        flow_state: &Flows<'a, 'tcx>,
+        term: &'a Terminator<'tcx>,
         loc: Location,
     ) {
         debug!("MirBorrowckCtxt::process_terminator({:?}, {:?}): {:?}", loc, term, flow_state);
@@ -794,8 +794,8 @@ impl<'a, 'mir, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'mir, 'tcx, R>
     fn visit_terminator_after_primary_effect(
         &mut self,
         _results: &mut R,
-        flow_state: &Flows<'_, 'mir, 'tcx>,
-        term: &'mir Terminator<'tcx>,
+        flow_state: &Flows<'a, 'tcx>,
+        term: &'a Terminator<'tcx>,
         loc: Location,
     ) {
         let span = term.source_info.span;
@@ -972,8 +972,8 @@ impl InitializationRequiringAction {
     }
 }
 
-impl<'mir, 'tcx> MirBorrowckCtxt<'_, 'mir, '_, 'tcx> {
-    fn body(&self) -> &'mir Body<'tcx> {
+impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> {
+    fn body(&self) -> &'a Body<'tcx> {
         self.body
     }
 
@@ -989,7 +989,7 @@ impl<'mir, 'tcx> MirBorrowckCtxt<'_, 'mir, '_, 'tcx> {
         place_span: (Place<'tcx>, Span),
         kind: (AccessDepth, ReadOrWrite),
         is_local_mutation_allowed: LocalMutationIsAllowed,
-        flow_state: &Flows<'_, 'mir, 'tcx>,
+        flow_state: &Flows<'a, 'tcx>,
     ) {
         let (sd, rw) = kind;
 
@@ -1039,7 +1039,7 @@ impl<'mir, 'tcx> MirBorrowckCtxt<'_, 'mir, '_, 'tcx> {
         place_span: (Place<'tcx>, Span),
         sd: AccessDepth,
         rw: ReadOrWrite,
-        flow_state: &Flows<'_, 'mir, 'tcx>,
+        flow_state: &Flows<'a, 'tcx>,
     ) -> bool {
         let mut error_reported = false;
         let borrow_set = Rc::clone(&self.borrow_set);
@@ -1180,7 +1180,7 @@ impl<'mir, 'tcx> MirBorrowckCtxt<'_, 'mir, '_, 'tcx> {
         location: Location,
         place_span: (Place<'tcx>, Span),
         kind: AccessDepth,
-        flow_state: &Flows<'_, 'mir, 'tcx>,
+        flow_state: &Flows<'a, 'tcx>,
     ) {
         // Write of P[i] or *P requires P init'd.
         self.check_if_assigned_path_is_moved(location, place_span, flow_state);
@@ -1197,8 +1197,8 @@ impl<'mir, 'tcx> MirBorrowckCtxt<'_, 'mir, '_, 'tcx> {
     fn consume_rvalue(
         &mut self,
         location: Location,
-        (rvalue, span): (&'mir Rvalue<'tcx>, Span),
-        flow_state: &Flows<'_, 'mir, 'tcx>,
+        (rvalue, span): (&'a Rvalue<'tcx>, Span),
+        flow_state: &Flows<'a, 'tcx>,
     ) {
         match rvalue {
             &Rvalue::Ref(_ /*rgn*/, bk, place) => {
@@ -1455,8 +1455,8 @@ impl<'mir, 'tcx> MirBorrowckCtxt<'_, 'mir, '_, 'tcx> {
     fn consume_operand(
         &mut self,
         location: Location,
-        (operand, span): (&'mir Operand<'tcx>, Span),
-        flow_state: &Flows<'_, 'mir, 'tcx>,
+        (operand, span): (&'a Operand<'tcx>, Span),
+        flow_state: &Flows<'a, 'tcx>,
     ) {
         match *operand {
             Operand::Copy(place) => {
@@ -1576,12 +1576,7 @@ impl<'mir, 'tcx> MirBorrowckCtxt<'_, 'mir, '_, 'tcx> {
         }
     }
 
-    fn check_activations(
-        &mut self,
-        location: Location,
-        span: Span,
-        flow_state: &Flows<'_, 'mir, 'tcx>,
-    ) {
+    fn check_activations(&mut self, location: Location, span: Span, flow_state: &Flows<'a, 'tcx>) {
         // Two-phase borrow support: For each activation that is newly
         // generated at this statement, check if it interferes with
         // another borrow.
@@ -1744,7 +1739,7 @@ impl<'mir, 'tcx> MirBorrowckCtxt<'_, 'mir, '_, 'tcx> {
         location: Location,
         desired_action: InitializationRequiringAction,
         place_span: (PlaceRef<'tcx>, Span),
-        flow_state: &Flows<'_, 'mir, 'tcx>,
+        flow_state: &Flows<'a, 'tcx>,
     ) {
         let maybe_uninits = &flow_state.uninits;
 
@@ -1849,7 +1844,7 @@ impl<'mir, 'tcx> MirBorrowckCtxt<'_, 'mir, '_, 'tcx> {
         location: Location,
         desired_action: InitializationRequiringAction,
         place_span: (PlaceRef<'tcx>, Span),
-        flow_state: &Flows<'_, 'mir, 'tcx>,
+        flow_state: &Flows<'a, 'tcx>,
     ) {
         let maybe_uninits = &flow_state.uninits;
 
@@ -1948,7 +1943,7 @@ impl<'mir, 'tcx> MirBorrowckCtxt<'_, 'mir, '_, 'tcx> {
         &mut self,
         location: Location,
         (place, span): (Place<'tcx>, Span),
-        flow_state: &Flows<'_, 'mir, 'tcx>,
+        flow_state: &Flows<'a, 'tcx>,
     ) {
         debug!("check_if_assigned_path_is_moved place: {:?}", place);
 
@@ -2009,12 +2004,12 @@ impl<'mir, 'tcx> MirBorrowckCtxt<'_, 'mir, '_, 'tcx> {
             }
         }
 
-        fn check_parent_of_field<'mir, 'tcx>(
-            this: &mut MirBorrowckCtxt<'_, 'mir, '_, 'tcx>,
+        fn check_parent_of_field<'a, 'tcx>(
+            this: &mut MirBorrowckCtxt<'a, '_, 'tcx>,
             location: Location,
             base: PlaceRef<'tcx>,
             span: Span,
-            flow_state: &Flows<'_, 'mir, 'tcx>,
+            flow_state: &Flows<'a, 'tcx>,
         ) {
             // rust-lang/rust#21232: Until Rust allows reads from the
             // initialized parts of partially initialized structs, we
@@ -2105,7 +2100,7 @@ impl<'mir, 'tcx> MirBorrowckCtxt<'_, 'mir, '_, 'tcx> {
         (place, span): (Place<'tcx>, Span),
         kind: ReadOrWrite,
         is_local_mutation_allowed: LocalMutationIsAllowed,
-        flow_state: &Flows<'_, 'mir, 'tcx>,
+        flow_state: &Flows<'a, 'tcx>,
         location: Location,
     ) -> bool {
         debug!(
@@ -2221,7 +2216,7 @@ impl<'mir, 'tcx> MirBorrowckCtxt<'_, 'mir, '_, 'tcx> {
     fn is_local_ever_initialized(
         &self,
         local: Local,
-        flow_state: &Flows<'_, 'mir, 'tcx>,
+        flow_state: &Flows<'a, 'tcx>,
     ) -> Option<InitIndex> {
         let mpi = self.move_data.rev_lookup.find_local(local)?;
         let ii = &self.move_data.init_path_map[mpi];
@@ -2229,7 +2224,7 @@ impl<'mir, 'tcx> MirBorrowckCtxt<'_, 'mir, '_, 'tcx> {
     }
 
     /// Adds the place into the used mutable variables set
-    fn add_used_mut(&mut self, root_place: RootPlace<'tcx>, flow_state: &Flows<'_, 'mir, 'tcx>) {
+    fn add_used_mut(&mut self, root_place: RootPlace<'tcx>, flow_state: &Flows<'a, 'tcx>) {
         match root_place {
             RootPlace { place_local: local, place_projection: [], is_local_mutation_allowed } => {
                 // If the local may have been initialized, and it is now currently being
@@ -2484,7 +2479,7 @@ mod diags {
         }
     }
 
-    impl<'infcx, 'tcx> MirBorrowckCtxt<'_, '_, 'infcx, 'tcx> {
+    impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
         pub(crate) fn buffer_error(&mut self, diag: Diag<'infcx>) {
             self.diags.buffer_error(diag);
         }
@@ -2522,7 +2517,7 @@ mod diags {
         }
 
         pub(crate) fn emit_errors(&mut self) -> Option<ErrorGuaranteed> {
-            let mut res = None;
+            let mut res = self.infcx.tainted_by_errors();
 
             // Buffer any move errors that we collected and de-duplicated.
             for (_, (_, diag)) in std::mem::take(&mut self.diags.buffered_move_errors) {
diff --git a/compiler/rustc_borrowck/src/nll.rs b/compiler/rustc_borrowck/src/nll.rs
index 6525befc13b..8575eb9e9a5 100644
--- a/compiler/rustc_borrowck/src/nll.rs
+++ b/compiler/rustc_borrowck/src/nll.rs
@@ -75,14 +75,14 @@ pub(crate) fn replace_regions_in_mir<'tcx>(
 /// Computes the (non-lexical) regions from the input MIR.
 ///
 /// This may result in errors being reported.
-pub(crate) fn compute_regions<'cx, 'tcx>(
+pub(crate) fn compute_regions<'a, 'tcx>(
     infcx: &BorrowckInferCtxt<'tcx>,
     universal_regions: UniversalRegions<'tcx>,
     body: &Body<'tcx>,
     promoted: &IndexSlice<Promoted, Body<'tcx>>,
     location_table: &LocationTable,
     param_env: ty::ParamEnv<'tcx>,
-    flow_inits: &mut ResultsCursor<'cx, 'tcx, MaybeInitializedPlaces<'_, 'cx, 'tcx>>,
+    flow_inits: &mut ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>>,
     move_data: &MoveData<'tcx>,
     borrow_set: &BorrowSet<'tcx>,
     upvars: &[&ty::CapturedPlace<'tcx>],
@@ -301,13 +301,13 @@ pub(super) fn dump_nll_mir<'tcx>(
 
 #[allow(rustc::diagnostic_outside_of_impl)]
 #[allow(rustc::untranslatable_diagnostic)]
-pub(super) fn dump_annotation<'tcx, 'cx>(
-    infcx: &'cx BorrowckInferCtxt<'tcx>,
+pub(super) fn dump_annotation<'tcx, 'infcx>(
+    infcx: &'infcx BorrowckInferCtxt<'tcx>,
     body: &Body<'tcx>,
     regioncx: &RegionInferenceContext<'tcx>,
     closure_region_requirements: &Option<ClosureRegionRequirements<'tcx>>,
     opaque_type_values: &FxIndexMap<LocalDefId, OpaqueHiddenType<'tcx>>,
-    diags: &mut crate::diags::BorrowckDiags<'cx, 'tcx>,
+    diags: &mut crate::diags::BorrowckDiags<'infcx, 'tcx>,
 ) {
     let tcx = infcx.tcx;
     let base_def_id = tcx.typeck_root_def_id(body.source.def_id());
diff --git a/compiler/rustc_borrowck/src/prefixes.rs b/compiler/rustc_borrowck/src/prefixes.rs
index 39d831378cd..aeb8a6c014a 100644
--- a/compiler/rustc_borrowck/src/prefixes.rs
+++ b/compiler/rustc_borrowck/src/prefixes.rs
@@ -34,7 +34,7 @@ pub(super) enum PrefixSet {
     Shallow,
 }
 
-impl<'tcx> MirBorrowckCtxt<'_, '_, '_, 'tcx> {
+impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
     /// Returns an iterator over the prefixes of `place`
     /// (inclusive) from longest to smallest, potentially
     /// terminating the iteration early based on `kind`.
diff --git a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
index a24fd95e3e6..79db8f4252b 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
@@ -30,11 +30,11 @@ mod trace;
 ///
 /// N.B., this computation requires normalization; therefore, it must be
 /// performed before
-pub(super) fn generate<'mir, 'tcx>(
+pub(super) fn generate<'a, 'tcx>(
     typeck: &mut TypeChecker<'_, 'tcx>,
     body: &Body<'tcx>,
     elements: &Rc<DenseLocationMap>,
-    flow_inits: &mut ResultsCursor<'mir, 'tcx, MaybeInitializedPlaces<'_, 'mir, 'tcx>>,
+    flow_inits: &mut ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>>,
     move_data: &MoveData<'tcx>,
 ) {
     debug!("liveness::generate");
diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
index 6186904e5fb..4d47863ae76 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
@@ -37,11 +37,11 @@ use crate::type_check::{NormalizeLocation, TypeChecker};
 /// DROP-LIVE set are to the liveness sets for regions found in the
 /// `dropck_outlives` result of the variable's type (in particular,
 /// this respects `#[may_dangle]` annotations).
-pub(super) fn trace<'mir, 'tcx>(
+pub(super) fn trace<'a, 'tcx>(
     typeck: &mut TypeChecker<'_, 'tcx>,
     body: &Body<'tcx>,
     elements: &Rc<DenseLocationMap>,
-    flow_inits: &mut ResultsCursor<'mir, 'tcx, MaybeInitializedPlaces<'_, 'mir, 'tcx>>,
+    flow_inits: &mut ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>>,
     move_data: &MoveData<'tcx>,
     relevant_live_locals: Vec<Local>,
     boring_locals: Vec<Local>,
@@ -99,29 +99,29 @@ pub(super) fn trace<'mir, 'tcx>(
 }
 
 /// Contextual state for the type-liveness coroutine.
-struct LivenessContext<'a, 'me, 'typeck, 'flow, 'tcx> {
+struct LivenessContext<'a, 'typeck, 'b, 'tcx> {
     /// Current type-checker, giving us our inference context etc.
-    typeck: &'me mut TypeChecker<'typeck, 'tcx>,
+    typeck: &'a mut TypeChecker<'typeck, 'tcx>,
 
     /// Defines the `PointIndex` mapping
-    elements: &'me DenseLocationMap,
+    elements: &'a DenseLocationMap,
 
     /// MIR we are analyzing.
-    body: &'me Body<'tcx>,
+    body: &'a Body<'tcx>,
 
     /// Mapping to/from the various indices used for initialization tracking.
-    move_data: &'me MoveData<'tcx>,
+    move_data: &'a MoveData<'tcx>,
 
     /// Cache for the results of `dropck_outlives` query.
     drop_data: FxIndexMap<Ty<'tcx>, DropData<'tcx>>,
 
     /// Results of dataflow tracking which variables (and paths) have been
     /// initialized.
-    flow_inits: &'me mut ResultsCursor<'flow, 'tcx, MaybeInitializedPlaces<'a, 'flow, 'tcx>>,
+    flow_inits: &'a mut ResultsCursor<'b, 'tcx, MaybeInitializedPlaces<'b, 'tcx>>,
 
     /// Index indicating where each variable is assigned, used, or
     /// dropped.
-    local_use_map: &'me LocalUseMap,
+    local_use_map: &'a LocalUseMap,
 }
 
 struct DropData<'tcx> {
@@ -129,8 +129,8 @@ struct DropData<'tcx> {
     region_constraint_data: Option<&'tcx QueryRegionConstraints<'tcx>>,
 }
 
-struct LivenessResults<'a, 'me, 'typeck, 'flow, 'tcx> {
-    cx: LivenessContext<'a, 'me, 'typeck, 'flow, 'tcx>,
+struct LivenessResults<'a, 'typeck, 'b, 'tcx> {
+    cx: LivenessContext<'a, 'typeck, 'b, 'tcx>,
 
     /// Set of points that define the current local.
     defs: BitSet<PointIndex>,
@@ -151,8 +151,8 @@ struct LivenessResults<'a, 'me, 'typeck, 'flow, 'tcx> {
     stack: Vec<PointIndex>,
 }
 
-impl<'a, 'me, 'typeck, 'flow, 'tcx> LivenessResults<'a, 'me, 'typeck, 'flow, 'tcx> {
-    fn new(cx: LivenessContext<'a, 'me, 'typeck, 'flow, 'tcx>) -> Self {
+impl<'a, 'typeck, 'b, 'tcx> LivenessResults<'a, 'typeck, 'b, 'tcx> {
+    fn new(cx: LivenessContext<'a, 'typeck, 'b, 'tcx>) -> Self {
         let num_points = cx.elements.num_points();
         LivenessResults {
             cx,
@@ -505,7 +505,7 @@ impl<'a, 'me, 'typeck, 'flow, 'tcx> LivenessResults<'a, 'me, 'typeck, 'flow, 'tc
     }
 }
 
-impl<'tcx> LivenessContext<'_, '_, '_, '_, 'tcx> {
+impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
     /// Returns `true` if the local variable (or some part of it) is initialized at the current
     /// cursor position. Callers should call one of the `seek` methods immediately before to point
     /// the cursor to the desired location.
diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs
index 224f8d5c893..3e5a6ad85fa 100644
--- a/compiler/rustc_borrowck/src/type_check/mod.rs
+++ b/compiler/rustc_borrowck/src/type_check/mod.rs
@@ -116,7 +116,7 @@ mod relate_tys;
 /// - `flow_inits` -- results of a maybe-init dataflow analysis
 /// - `move_data` -- move-data constructed when performing the maybe-init dataflow analysis
 /// - `elements` -- MIR region map
-pub(crate) fn type_check<'mir, 'tcx>(
+pub(crate) fn type_check<'a, 'tcx>(
     infcx: &BorrowckInferCtxt<'tcx>,
     param_env: ty::ParamEnv<'tcx>,
     body: &Body<'tcx>,
@@ -125,7 +125,7 @@ pub(crate) fn type_check<'mir, 'tcx>(
     location_table: &LocationTable,
     borrow_set: &BorrowSet<'tcx>,
     all_facts: &mut Option<AllFacts>,
-    flow_inits: &mut ResultsCursor<'mir, 'tcx, MaybeInitializedPlaces<'_, 'mir, 'tcx>>,
+    flow_inits: &mut ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>>,
     move_data: &MoveData<'tcx>,
     elements: &Rc<DenseLocationMap>,
     upvars: &[&ty::CapturedPlace<'tcx>],
@@ -1979,19 +1979,76 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
 
                 match cast_kind {
                     CastKind::PointerCoercion(PointerCoercion::ReifyFnPointer) => {
-                        let fn_sig = op.ty(body, tcx).fn_sig(tcx);
+                        let src_sig = op.ty(body, tcx).fn_sig(tcx);
+
+                        // HACK: This shouldn't be necessary... We can remove this when we actually
+                        // get binders with where clauses, then elaborate implied bounds into that
+                        // binder, and implement a higher-ranked subtyping algorithm that actually
+                        // respects these implied bounds.
+                        //
+                        // This protects against the case where we are casting from a higher-ranked
+                        // fn item to a non-higher-ranked fn pointer, where the cast throws away
+                        // implied bounds that would've needed to be checked at the call site. This
+                        // only works when we're casting to a non-higher-ranked fn ptr, since
+                        // placeholders in the target signature could have untracked implied
+                        // bounds, resulting in incorrect errors.
+                        //
+                        // We check that this signature is WF before subtyping the signature with
+                        // the target fn sig.
+                        if src_sig.has_bound_regions()
+                            && let ty::FnPtr(target_fn_tys, target_hdr) = *ty.kind()
+                            && let target_sig = target_fn_tys.with(target_hdr)
+                            && let Some(target_sig) = target_sig.no_bound_vars()
+                        {
+                            let src_sig = self.infcx.instantiate_binder_with_fresh_vars(
+                                span,
+                                BoundRegionConversionTime::HigherRankedType,
+                                src_sig,
+                            );
+                            let src_ty = Ty::new_fn_ptr(self.tcx(), ty::Binder::dummy(src_sig));
+                            self.prove_predicate(
+                                ty::ClauseKind::WellFormed(src_ty.into()),
+                                location.to_locations(),
+                                ConstraintCategory::Cast { unsize_to: None },
+                            );
+
+                            let src_ty = self.normalize(src_ty, location);
+                            if let Err(terr) = self.sub_types(
+                                src_ty,
+                                *ty,
+                                location.to_locations(),
+                                ConstraintCategory::Cast { unsize_to: None },
+                            ) {
+                                span_mirbug!(
+                                    self,
+                                    rvalue,
+                                    "equating {:?} with {:?} yields {:?}",
+                                    target_sig,
+                                    src_sig,
+                                    terr
+                                );
+                            };
+                        }
+
+                        let src_ty = Ty::new_fn_ptr(tcx, src_sig);
+                        // HACK: We want to assert that the signature of the source fn is
+                        // well-formed, because we don't enforce that via the WF of FnDef
+                        // types normally. This should be removed when we improve the tracking
+                        // of implied bounds of fn signatures.
+                        self.prove_predicate(
+                            ty::ClauseKind::WellFormed(src_ty.into()),
+                            location.to_locations(),
+                            ConstraintCategory::Cast { unsize_to: None },
+                        );
 
                         // The type that we see in the fcx is like
                         // `foo::<'a, 'b>`, where `foo` is the path to a
                         // function definition. When we extract the
                         // signature, it comes from the `fn_sig` query,
                         // and hence may contain unnormalized results.
-                        let fn_sig = self.normalize(fn_sig, location);
-
-                        let ty_fn_ptr_from = Ty::new_fn_ptr(tcx, fn_sig);
-
+                        let src_ty = self.normalize(src_ty, location);
                         if let Err(terr) = self.sub_types(
-                            ty_fn_ptr_from,
+                            src_ty,
                             *ty,
                             location.to_locations(),
                             ConstraintCategory::Cast { unsize_to: None },
@@ -2000,7 +2057,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                                 self,
                                 rvalue,
                                 "equating {:?} with {:?} yields {:?}",
-                                ty_fn_ptr_from,
+                                src_ty,
                                 ty,
                                 terr
                             );
diff --git a/compiler/rustc_borrowck/src/universal_regions.rs b/compiler/rustc_borrowck/src/universal_regions.rs
index 9eabe817359..c3edbcb50cc 100644
--- a/compiler/rustc_borrowck/src/universal_regions.rs
+++ b/compiler/rustc_borrowck/src/universal_regions.rs
@@ -29,7 +29,8 @@ use rustc_macros::extension;
 use rustc_middle::ty::fold::TypeFoldable;
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::{
-    self, GenericArgs, GenericArgsRef, InlineConstArgs, InlineConstArgsParts, RegionVid, Ty, TyCtxt,
+    self, GenericArgs, GenericArgsRef, InlineConstArgs, InlineConstArgsParts, RegionVid, Ty,
+    TyCtxt, TypeVisitableExt,
 };
 use rustc_middle::{bug, span_bug};
 use rustc_span::symbol::{kw, sym};
@@ -422,8 +423,8 @@ impl<'tcx> UniversalRegions<'tcx> {
     }
 }
 
-struct UniversalRegionsBuilder<'cx, 'tcx> {
-    infcx: &'cx BorrowckInferCtxt<'tcx>,
+struct UniversalRegionsBuilder<'infcx, 'tcx> {
+    infcx: &'infcx BorrowckInferCtxt<'tcx>,
     mir_def: LocalDefId,
     param_env: ty::ParamEnv<'tcx>,
 }
@@ -688,7 +689,8 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
         defining_ty: DefiningTy<'tcx>,
     ) -> ty::Binder<'tcx, &'tcx ty::List<Ty<'tcx>>> {
         let tcx = self.infcx.tcx;
-        match defining_ty {
+
+        let inputs_and_output = match defining_ty {
             DefiningTy::Closure(def_id, args) => {
                 assert_eq!(self.mir_def.to_def_id(), def_id);
                 let closure_sig = args.as_closure().sig();
@@ -798,6 +800,7 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
                 // "output" (the type of the constant).
                 assert_eq!(self.mir_def.to_def_id(), def_id);
                 let ty = tcx.type_of(self.mir_def).instantiate_identity();
+
                 let ty = indices.fold_to_region_vids(tcx, ty);
                 ty::Binder::dummy(tcx.mk_type_list(&[ty]))
             }
@@ -807,7 +810,14 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
                 let ty = args.as_inline_const().ty();
                 ty::Binder::dummy(tcx.mk_type_list(&[ty]))
             }
+        };
+
+        // FIXME(#129952): We probably want a more principled approach here.
+        if let Err(terr) = inputs_and_output.skip_binder().error_reported() {
+            self.infcx.set_tainted_by_errors(terr);
         }
+
+        inputs_and_output
     }
 }
 
diff --git a/compiler/rustc_borrowck/src/used_muts.rs b/compiler/rustc_borrowck/src/used_muts.rs
index 981e7daf670..bde07c05c0e 100644
--- a/compiler/rustc_borrowck/src/used_muts.rs
+++ b/compiler/rustc_borrowck/src/used_muts.rs
@@ -7,7 +7,7 @@ use tracing::debug;
 
 use crate::MirBorrowckCtxt;
 
-impl<'tcx> MirBorrowckCtxt<'_, '_, '_, 'tcx> {
+impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
     /// Walks the MIR adding to the set of `used_mut` locals that will be ignored for the purposes
     /// of the `unused_mut` lint.
     ///
@@ -46,13 +46,13 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, '_, 'tcx> {
 
 /// MIR visitor for collecting used mutable variables.
 /// The 'visit lifetime represents the duration of the MIR walk.
-struct GatherUsedMutsVisitor<'visit, 'a, 'mir, 'infcx, 'tcx> {
+struct GatherUsedMutsVisitor<'a, 'b, 'infcx, 'tcx> {
     temporary_used_locals: FxIndexSet<Local>,
-    never_initialized_mut_locals: &'visit mut FxIndexSet<Local>,
-    mbcx: &'visit mut MirBorrowckCtxt<'a, 'mir, 'infcx, 'tcx>,
+    never_initialized_mut_locals: &'a mut FxIndexSet<Local>,
+    mbcx: &'a mut MirBorrowckCtxt<'b, 'infcx, 'tcx>,
 }
 
-impl GatherUsedMutsVisitor<'_, '_, '_, '_, '_> {
+impl GatherUsedMutsVisitor<'_, '_, '_, '_> {
     fn remove_never_initialized_mut_locals(&mut self, into: Place<'_>) {
         // Remove any locals that we found were initialized from the
         // `never_initialized_mut_locals` set. At the end, the only remaining locals will
@@ -64,7 +64,7 @@ impl GatherUsedMutsVisitor<'_, '_, '_, '_, '_> {
     }
 }
 
-impl<'tcx> Visitor<'tcx> for GatherUsedMutsVisitor<'_, '_, '_, '_, 'tcx> {
+impl<'tcx> Visitor<'tcx> for GatherUsedMutsVisitor<'_, '_, '_, 'tcx> {
     fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) {
         debug!("visit_terminator: terminator={:?}", terminator);
         match &terminator.kind {
diff --git a/compiler/rustc_codegen_cranelift/Readme.md b/compiler/rustc_codegen_cranelift/Readme.md
index 6766e2f844d..18a840f8a50 100644
--- a/compiler/rustc_codegen_cranelift/Readme.md
+++ b/compiler/rustc_codegen_cranelift/Readme.md
@@ -70,7 +70,7 @@ For more docs on how to build and test see [build_system/usage.txt](build_system
 |AIX|❌[^xcoff]|N/A|N/A|❌[^xcoff]|
 |Other unixes|❓|❓|❓|❓|
 |macOS|✅|✅|N/A|N/A|
-|Windows|✅[^no-rustup]|❌|N/A|N/A|
+|Windows|✅|❌|N/A|N/A|
 
 ✅: Fully supported and tested
 ❓: Maybe supported, not tested
diff --git a/compiler/rustc_codegen_gcc/Cargo.lock b/compiler/rustc_codegen_gcc/Cargo.lock
index 915229f7e7e..0eb264f99eb 100644
--- a/compiler/rustc_codegen_gcc/Cargo.lock
+++ b/compiler/rustc_codegen_gcc/Cargo.lock
@@ -12,63 +12,12 @@ dependencies = [
 ]
 
 [[package]]
-name = "bitflags"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
-
-[[package]]
-name = "bitflags"
-version = "2.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
-
-[[package]]
 name = "boml"
 version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "85fdb93f04c73bff54305fa437ffea5449c41edcaadfe882f35836206b166ac5"
 
 [[package]]
-name = "cc"
-version = "1.0.79"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
-
-[[package]]
-name = "cfg-if"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
-
-[[package]]
-name = "errno"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
-dependencies = [
- "errno-dragonfly",
- "libc",
- "windows-sys",
-]
-
-[[package]]
-name = "errno-dragonfly"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
-dependencies = [
- "cc",
- "libc",
-]
-
-[[package]]
-name = "fastrand"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764"
-
-[[package]]
 name = "fm"
 version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -133,12 +82,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c"
 
 [[package]]
-name = "linux-raw-sys"
-version = "0.4.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503"
-
-[[package]]
 name = "memchr"
 version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -155,24 +98,6 @@ dependencies = [
 ]
 
 [[package]]
-name = "object"
-version = "0.30.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03b4680b86d9cfafba8fc491dc9b6df26b68cf40e9e6cd73909194759a63c385"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "redox_syscall"
-version = "0.3.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
-dependencies = [
- "bitflags 1.3.2",
-]
-
-[[package]]
 name = "regex"
 version = "1.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -196,22 +121,6 @@ dependencies = [
  "boml",
  "gccjit",
  "lang_tester",
- "object",
- "smallvec",
- "tempfile",
-]
-
-[[package]]
-name = "rustix"
-version = "0.38.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f"
-dependencies = [
- "bitflags 2.4.0",
- "errno",
- "libc",
- "linux-raw-sys",
- "windows-sys",
 ]
 
 [[package]]
@@ -224,25 +133,6 @@ dependencies = [
 ]
 
 [[package]]
-name = "smallvec"
-version = "1.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
-
-[[package]]
-name = "tempfile"
-version = "3.7.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651"
-dependencies = [
- "cfg-if",
- "fastrand",
- "redox_syscall",
- "rustix",
- "windows-sys",
-]
-
-[[package]]
 name = "termcolor"
 version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -315,69 +205,3 @@ name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
-
-[[package]]
-name = "windows-sys"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
-dependencies = [
- "windows-targets",
-]
-
-[[package]]
-name = "windows-targets"
-version = "0.48.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f"
-dependencies = [
- "windows_aarch64_gnullvm",
- "windows_aarch64_msvc",
- "windows_i686_gnu",
- "windows_i686_msvc",
- "windows_x86_64_gnu",
- "windows_x86_64_gnullvm",
- "windows_x86_64_msvc",
-]
-
-[[package]]
-name = "windows_aarch64_gnullvm"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
-
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
-
-[[package]]
-name = "windows_i686_gnu"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
-
-[[package]]
-name = "windows_i686_msvc"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
-
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
-
-[[package]]
-name = "windows_x86_64_gnullvm"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
-
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
diff --git a/compiler/rustc_codegen_gcc/Cargo.toml b/compiler/rustc_codegen_gcc/Cargo.toml
index 5caca63f634..44fb5eef04a 100644
--- a/compiler/rustc_codegen_gcc/Cargo.toml
+++ b/compiler/rustc_codegen_gcc/Cargo.toml
@@ -23,21 +23,11 @@ default = ["master"]
 
 [dependencies]
 gccjit = "2.1"
-
 # Local copy.
 #gccjit = { path = "../gccjit.rs" }
 
-object = { version = "0.30.1", default-features = false, features = [
-    "std",
-    "read",
-] }
-smallvec = { version = "1.6.1", features = ["union", "may_dangle"] }
-# TODO(antoyo): make tempfile optional.
-tempfile = "3.7.1"
-
 [dev-dependencies]
 lang_tester = "0.8.0"
-tempfile = "3.1.0"
 boml = "0.3.1"
 
 [profile.dev]
diff --git a/compiler/rustc_codegen_gcc/src/lib.rs b/compiler/rustc_codegen_gcc/src/lib.rs
index 94f016234f9..4de671ac4a0 100644
--- a/compiler/rustc_codegen_gcc/src/lib.rs
+++ b/compiler/rustc_codegen_gcc/src/lib.rs
@@ -24,6 +24,14 @@
 #![deny(clippy::pattern_type_mismatch)]
 #![allow(clippy::needless_lifetimes)]
 
+// Some "regular" crates we want to share with rustc
+extern crate object;
+extern crate smallvec;
+extern crate tempfile;
+#[macro_use]
+extern crate tracing;
+
+// The rustc crates we need
 extern crate rustc_apfloat;
 extern crate rustc_ast;
 extern crate rustc_attr;
@@ -42,8 +50,6 @@ extern crate rustc_middle;
 extern crate rustc_session;
 extern crate rustc_span;
 extern crate rustc_target;
-#[macro_use]
-extern crate tracing;
 
 // This prevents duplicating functions and statics that are already part of the host rustc process.
 #[allow(unused_extern_crates)]
diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs
index 92a857c2adc..9d4497d73a8 100644
--- a/compiler/rustc_codegen_llvm/src/attributes.rs
+++ b/compiler/rustc_codegen_llvm/src/attributes.rs
@@ -411,26 +411,31 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
             // the string "false". Now it is disabled by absence of the attribute.
             to_add.push(llvm::CreateAttrStringValue(cx.llcx, "branch-target-enforcement", "false"));
         }
-    } else if llvm_util::get_version() >= (19, 0, 0) {
-        // For non-naked functions, set branch protection attributes on aarch64.
-        if let Some(BranchProtection { bti, pac_ret }) =
-            cx.sess().opts.unstable_opts.branch_protection
-        {
-            assert!(cx.sess().target.arch == "aarch64");
-            if bti {
-                to_add.push(llvm::CreateAttrString(cx.llcx, "branch-target-enforcement"));
-            }
-            if let Some(PacRet { leaf, key }) = pac_ret {
-                to_add.push(llvm::CreateAttrStringValue(
-                    cx.llcx,
-                    "sign-return-address",
-                    if leaf { "all" } else { "non-leaf" },
-                ));
-                to_add.push(llvm::CreateAttrStringValue(
-                    cx.llcx,
-                    "sign-return-address-key",
-                    if key == PAuthKey::A { "a_key" } else { "b_key" },
-                ));
+    } else {
+        // Do not set sanitizer attributes for naked functions.
+        to_add.extend(sanitize_attrs(cx, codegen_fn_attrs.no_sanitize));
+
+        if llvm_util::get_version() >= (19, 0, 0) {
+            // For non-naked functions, set branch protection attributes on aarch64.
+            if let Some(BranchProtection { bti, pac_ret }) =
+                cx.sess().opts.unstable_opts.branch_protection
+            {
+                assert!(cx.sess().target.arch == "aarch64");
+                if bti {
+                    to_add.push(llvm::CreateAttrString(cx.llcx, "branch-target-enforcement"));
+                }
+                if let Some(PacRet { leaf, key }) = pac_ret {
+                    to_add.push(llvm::CreateAttrStringValue(
+                        cx.llcx,
+                        "sign-return-address",
+                        if leaf { "all" } else { "non-leaf" },
+                    ));
+                    to_add.push(llvm::CreateAttrStringValue(
+                        cx.llcx,
+                        "sign-return-address-key",
+                        if key == PAuthKey::A { "a_key" } else { "b_key" },
+                    ));
+                }
             }
         }
     }
@@ -485,7 +490,6 @@ pub(crate) fn llfn_attrs_from_instance<'ll, 'tcx>(
     if let Some(backchain) = backchain_attr(cx) {
         to_add.push(backchain);
     }
-    to_add.extend(sanitize_attrs(cx, codegen_fn_attrs.no_sanitize));
     to_add.extend(patchable_function_entry_attrs(cx, codegen_fn_attrs.patchable_function_entry));
 
     // Always annotate functions with the target-cpu they are compiled for.
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
index 17a9630c655..d231b103964 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
@@ -456,7 +456,7 @@ pub(crate) fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) ->
             if def.is_box()
                 && args.get(1).map_or(true, |arg| cx.layout_of(arg.expect_ty()).is_1zst()) =>
         {
-            build_pointer_or_reference_di_node(cx, t, t.boxed_ty(), unique_type_id)
+            build_pointer_or_reference_di_node(cx, t, t.expect_boxed_ty(), unique_type_id)
         }
         ty::FnDef(..) | ty::FnPtr(..) => build_subroutine_type_di_node(cx, unique_type_id),
         ty::Closure(..) => build_closure_env_di_node(cx, unique_type_id),
diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs
index 6a303e1e602..7f26bbd7f87 100644
--- a/compiler/rustc_codegen_llvm/src/lib.rs
+++ b/compiler/rustc_codegen_llvm/src/lib.rs
@@ -6,7 +6,6 @@
 
 // tidy-alphabetical-start
 #![allow(internal_features)]
-#![cfg_attr(bootstrap, feature(unsafe_extern_blocks))]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(assert_matches)]
diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs
index 618602ed70f..d55220ba5c3 100644
--- a/compiler/rustc_codegen_llvm/src/llvm_util.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs
@@ -353,9 +353,7 @@ pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> {
                 None
             }
         })
-        .filter(|feature| {
-            RUSTC_SPECIAL_FEATURES.contains(feature) || features.contains(&Symbol::intern(feature))
-        })
+        .filter(|feature| features.contains(&Symbol::intern(feature)))
         .map(|feature| Symbol::intern(feature))
         .collect()
 }
diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs
index 9b5a797ad51..0fd9d7fffe8 100644
--- a/compiler/rustc_codegen_ssa/src/back/metadata.rs
+++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs
@@ -372,27 +372,42 @@ pub(crate) fn create_object_file(sess: &Session) -> Option<write::Object<'static
     Some(file)
 }
 
-/// Since Xcode 15 Apple's LD requires object files to contain information about what they were
-/// built for (LC_BUILD_VERSION): the platform (macOS/watchOS etc), minimum OS version, and SDK
-/// version. This returns a `MachOBuildVersion` for the target.
+/// Mach-O files contain information about:
+/// - The platform/OS they were built for (macOS/watchOS/Mac Catalyst/iOS simulator etc).
+/// - The minimum OS version / deployment target.
+/// - The version of the SDK they were targetting.
+///
+/// In the past, this was accomplished using the LC_VERSION_MIN_MACOSX, LC_VERSION_MIN_IPHONEOS,
+/// LC_VERSION_MIN_TVOS or LC_VERSION_MIN_WATCHOS load commands, which each contain information
+/// about the deployment target and SDK version, and implicitly, by their presence, which OS they
+/// target. Simulator targets were determined if the architecture was x86_64, but there was e.g. a
+/// LC_VERSION_MIN_IPHONEOS present.
+///
+/// This is of course brittle and limited, so modern tooling emit the LC_BUILD_VERSION load
+/// command (which contains all three pieces of information in one) when the deployment target is
+/// high enough, or the target is something that wouldn't be encodable with the old load commands
+/// (such as Mac Catalyst, or Aarch64 iOS simulator).
+///
+/// Since Xcode 15, Apple's LD apparently requires object files to use this load command, so this
+/// returns the `MachOBuildVersion` for the target to do so.
 fn macho_object_build_version_for_target(target: &Target) -> object::write::MachOBuildVersion {
     /// The `object` crate demands "X.Y.Z encoded in nibbles as xxxx.yy.zz"
     /// e.g. minOS 14.0 = 0x000E0000, or SDK 16.2 = 0x00100200
-    fn pack_version((major, minor): (u32, u32)) -> u32 {
-        (major << 16) | (minor << 8)
+    fn pack_version((major, minor, patch): (u16, u8, u8)) -> u32 {
+        let (major, minor, patch) = (major as u32, minor as u32, patch as u32);
+        (major << 16) | (minor << 8) | patch
     }
 
     let platform =
         rustc_target::spec::current_apple_platform(target).expect("unknown Apple target OS");
-    let min_os = rustc_target::spec::current_apple_deployment_target(target)
-        .expect("unknown Apple target OS");
-    let sdk =
+    let min_os = rustc_target::spec::current_apple_deployment_target(target);
+    let (sdk_major, sdk_minor) =
         rustc_target::spec::current_apple_sdk_version(platform).expect("unknown Apple target OS");
 
     let mut build_version = object::write::MachOBuildVersion::default();
     build_version.platform = platform;
     build_version.minos = pack_version(min_os);
-    build_version.sdk = pack_version(sdk);
+    build_version.sdk = pack_version((sdk_major, sdk_minor, 0));
     build_version
 }
 
diff --git a/compiler/rustc_const_eval/src/check_consts/check.rs b/compiler/rustc_const_eval/src/check_consts/check.rs
index 8be327a8b56..828d8354c63 100644
--- a/compiler/rustc_const_eval/src/check_consts/check.rs
+++ b/compiler/rustc_const_eval/src/check_consts/check.rs
@@ -538,8 +538,9 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                             // final value.
                             // Note: This is only sound if every local that has a `StorageDead` has a
                             // `StorageDead` in every control flow path leading to a `return` terminator.
-                            // The good news is that interning will detect if any unexpected mutable
-                            // pointer slips through.
+                            // If anything slips through, there's no safety net -- safe code can create
+                            // references to variants of `!Freeze` enums as long as that variant is `Freeze`,
+                            // so interning can't protect us here.
                             if self.local_is_transient(place.local) {
                                 self.check_op(ops::TransientCellBorrow);
                             } else {
diff --git a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
index 96b3ec6f187..594b98f34ef 100644
--- a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
+++ b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
@@ -10,7 +10,6 @@ use rustc_middle::traits::Reveal;
 use rustc_middle::ty::layout::LayoutOf;
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_session::lint;
 use rustc_span::def_id::LocalDefId;
 use rustc_span::{Span, DUMMY_SP};
 use rustc_target::abi::{self, Abi};
@@ -18,13 +17,12 @@ use tracing::{debug, instrument, trace};
 
 use super::{CanAccessMutGlobal, CompileTimeInterpCx, CompileTimeMachine};
 use crate::const_eval::CheckAlignment;
-use crate::errors::{self, ConstEvalError, DanglingPtrInFinal};
 use crate::interpret::{
     create_static_alloc, eval_nullary_intrinsic, intern_const_alloc_recursive, throw_exhaust,
     CtfeValidationMode, GlobalId, Immediate, InternKind, InternResult, InterpCx, InterpError,
     InterpResult, MPlaceTy, MemoryKind, OpTy, RefTracking, StackPopCleanup,
 };
-use crate::CTRL_C_RECEIVED;
+use crate::{errors, CTRL_C_RECEIVED};
 
 // Returns a pointer to where the result lives
 #[instrument(level = "trace", skip(ecx, body))]
@@ -94,7 +92,7 @@ fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
     let intern_result = intern_const_alloc_recursive(ecx, intern_kind, &ret);
 
     // Since evaluation had no errors, validate the resulting constant.
-    const_validate_mplace(&ecx, &ret, cid)?;
+    const_validate_mplace(ecx, &ret, cid)?;
 
     // Only report this after validation, as validaiton produces much better diagnostics.
     // FIXME: ensure validation always reports this and stop making interning care about it.
@@ -105,18 +103,15 @@ fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
             return Err(ecx
                 .tcx
                 .dcx()
-                .emit_err(DanglingPtrInFinal { span: ecx.tcx.span, kind: intern_kind })
+                .emit_err(errors::DanglingPtrInFinal { span: ecx.tcx.span, kind: intern_kind })
                 .into());
         }
         Err(InternResult::FoundBadMutablePointer) => {
-            // only report mutable pointers if there were no dangling pointers
-            let err_diag = errors::MutablePtrInFinal { span: ecx.tcx.span, kind: intern_kind };
-            ecx.tcx.emit_node_span_lint(
-                lint::builtin::CONST_EVAL_MUTABLE_PTR_IN_FINAL_VALUE,
-                ecx.machine.best_lint_scope(*ecx.tcx),
-                err_diag.span,
-                err_diag,
-            )
+            return Err(ecx
+                .tcx
+                .dcx()
+                .emit_err(errors::MutablePtrInFinal { span: ecx.tcx.span, kind: intern_kind })
+                .into());
         }
     }
 
@@ -391,7 +386,7 @@ fn eval_in_interpreter<'tcx, R: InterpretationResult<'tcx>>(
 
 #[inline(always)]
 fn const_validate_mplace<'tcx>(
-    ecx: &InterpCx<'tcx, CompileTimeMachine<'tcx>>,
+    ecx: &mut InterpCx<'tcx, CompileTimeMachine<'tcx>>,
     mplace: &MPlaceTy<'tcx>,
     cid: GlobalId<'tcx>,
 ) -> Result<(), ErrorHandled> {
@@ -448,7 +443,12 @@ fn report_eval_error<'tcx>(
         error,
         DUMMY_SP,
         || super::get_span_and_frames(ecx.tcx, ecx.stack()),
-        |span, frames| ConstEvalError { span, error_kind: kind, instance, frame_notes: frames },
+        |span, frames| errors::ConstEvalError {
+            span,
+            error_kind: kind,
+            instance,
+            frame_notes: frames,
+        },
     )
 }
 
diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs
index 9c1fef095f5..6a691abae02 100644
--- a/compiler/rustc_const_eval/src/const_eval/machine.rs
+++ b/compiler/rustc_const_eval/src/const_eval/machine.rs
@@ -1,16 +1,16 @@
-use std::borrow::Borrow;
+use std::borrow::{Borrow, Cow};
 use std::fmt;
 use std::hash::Hash;
 use std::ops::ControlFlow;
 
 use rustc_ast::Mutability;
-use rustc_data_structures::fx::{FxIndexMap, IndexEntry};
+use rustc_data_structures::fx::{FxHashMap, FxIndexMap, IndexEntry};
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::{self as hir, LangItem, CRATE_HIR_ID};
 use rustc_middle::mir::AssertMessage;
 use rustc_middle::query::TyCtxtAt;
 use rustc_middle::ty::layout::{FnAbiOf, TyAndLayout};
-use rustc_middle::ty::{self, TyCtxt};
+use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_middle::{bug, mir};
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::Span;
@@ -24,8 +24,8 @@ use crate::fluent_generated as fluent;
 use crate::interpret::{
     self, compile_time_machine, err_ub, throw_exhaust, throw_inval, throw_ub_custom, throw_unsup,
     throw_unsup_format, AllocId, AllocRange, ConstAllocation, CtfeProvenance, FnArg, Frame,
-    GlobalAlloc, ImmTy, InterpCx, InterpResult, MPlaceTy, OpTy, Pointer, PointerArithmetic, Scalar,
-    StackPopCleanup,
+    GlobalAlloc, ImmTy, InterpCx, InterpResult, MPlaceTy, OpTy, Pointer, PointerArithmetic,
+    RangeSet, Scalar, StackPopCleanup,
 };
 
 /// When hitting this many interpreted terminators we emit a deny by default lint
@@ -65,6 +65,9 @@ pub struct CompileTimeMachine<'tcx> {
     /// storing the result in the given `AllocId`.
     /// Used to prevent reads from a static's base allocation, as that may allow for self-initialization loops.
     pub(crate) static_root_ids: Option<(AllocId, LocalDefId)>,
+
+    /// A cache of "data range" computations for unions (i.e., the offsets of non-padding bytes).
+    union_data_ranges: FxHashMap<Ty<'tcx>, RangeSet>,
 }
 
 #[derive(Copy, Clone)]
@@ -99,6 +102,7 @@ impl<'tcx> CompileTimeMachine<'tcx> {
             can_access_mut_global,
             check_alignment,
             static_root_ids: None,
+            union_data_ranges: FxHashMap::default(),
         }
     }
 }
@@ -714,16 +718,29 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
         _kind: mir::RetagKind,
         val: &ImmTy<'tcx, CtfeProvenance>,
     ) -> InterpResult<'tcx, ImmTy<'tcx, CtfeProvenance>> {
-        // If it's a frozen shared reference that's not already immutable, make it immutable.
+        // If it's a frozen shared reference that's not already immutable, potentially make it immutable.
         // (Do nothing on `None` provenance, that cannot store immutability anyway.)
         if let ty::Ref(_, ty, mutbl) = val.layout.ty.kind()
             && *mutbl == Mutability::Not
-            && val.to_scalar_and_meta().0.to_pointer(ecx)?.provenance.is_some_and(|p| !p.immutable())
-            // That next check is expensive, that's why we have all the guards above.
-            && ty.is_freeze(*ecx.tcx, ecx.param_env)
+            && val
+                .to_scalar_and_meta()
+                .0
+                .to_pointer(ecx)?
+                .provenance
+                .is_some_and(|p| !p.immutable())
         {
+            // That next check is expensive, that's why we have all the guards above.
+            let is_immutable = ty.is_freeze(*ecx.tcx, ecx.param_env);
             let place = ecx.ref_to_mplace(val)?;
-            let new_place = place.map_provenance(CtfeProvenance::as_immutable);
+            let new_place = if is_immutable {
+                place.map_provenance(CtfeProvenance::as_immutable)
+            } else {
+                // Even if it is not immutable, remember that it is a shared reference.
+                // This allows it to become part of the final value of the constant.
+                // (See <https://github.com/rust-lang/rust/pull/128543> for why we allow this
+                // even when there is interior mutability.)
+                place.map_provenance(CtfeProvenance::as_shared_ref)
+            };
             Ok(ImmTy::from_immediate(new_place.to_ref(ecx), val.layout))
         } else {
             Ok(val.clone())
@@ -766,6 +783,19 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
         }
         Ok(())
     }
+
+    fn cached_union_data_range<'e>(
+        ecx: &'e mut InterpCx<'tcx, Self>,
+        ty: Ty<'tcx>,
+        compute_range: impl FnOnce() -> RangeSet,
+    ) -> Cow<'e, RangeSet> {
+        if ecx.tcx.sess.opts.unstable_opts.extra_const_ub_checks {
+            Cow::Borrowed(ecx.machine.union_data_ranges.entry(ty).or_insert_with(compute_range))
+        } else {
+            // Don't bother caching, we're only doing one validation at the end anyway.
+            Cow::Owned(compute_range())
+        }
+    }
 }
 
 // Please do not add any code below the above `Machine` trait impl. I (oli-obk) plan more cleanups
diff --git a/compiler/rustc_const_eval/src/errors.rs b/compiler/rustc_const_eval/src/errors.rs
index 0b366b43f95..57e52254757 100644
--- a/compiler/rustc_const_eval/src/errors.rs
+++ b/compiler/rustc_const_eval/src/errors.rs
@@ -35,13 +35,10 @@ pub(crate) struct NestedStaticInThreadLocal {
     pub span: Span,
 }
 
-#[derive(LintDiagnostic)]
+#[derive(Diagnostic)]
 #[diag(const_eval_mutable_ptr_in_final)]
 pub(crate) struct MutablePtrInFinal {
-    // rust-lang/rust#122153: This was marked as `#[primary_span]` under
-    // `derive(Diagnostic)`. Since we expect we may hard-error in future, we are
-    // keeping the field (and skipping it under `derive(LintDiagnostic)`).
-    #[skip_arg]
+    #[primary_span]
     pub span: Span,
     pub kind: InternKind,
 }
diff --git a/compiler/rustc_const_eval/src/interpret/call.rs b/compiler/rustc_const_eval/src/interpret/call.rs
index 82438eb5e78..568a9a3a637 100644
--- a/compiler/rustc_const_eval/src/interpret/call.rs
+++ b/compiler/rustc_const_eval/src/interpret/call.rs
@@ -189,7 +189,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 ty::Ref(_, ty, _) => *ty,
                 ty::RawPtr(ty, _) => *ty,
                 // We only accept `Box` with the default allocator.
-                _ if ty.is_box_global(*self.tcx) => ty.boxed_ty(),
+                _ if ty.is_box_global(*self.tcx) => ty.expect_boxed_ty(),
                 _ => return Ok(None),
             }))
         };
diff --git a/compiler/rustc_const_eval/src/interpret/discriminant.rs b/compiler/rustc_const_eval/src/interpret/discriminant.rs
index 0008a15722b..de93ed85704 100644
--- a/compiler/rustc_const_eval/src/interpret/discriminant.rs
+++ b/compiler/rustc_const_eval/src/interpret/discriminant.rs
@@ -7,7 +7,7 @@ use rustc_target::abi::{self, TagEncoding, VariantIdx, Variants};
 use tracing::{instrument, trace};
 
 use super::{
-    err_ub, throw_ub, ImmTy, InterpCx, InterpResult, Machine, Readable, Scalar, Writeable,
+    err_ub, throw_ub, ImmTy, InterpCx, InterpResult, Machine, Projectable, Scalar, Writeable,
 };
 
 impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
@@ -60,7 +60,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     #[instrument(skip(self), level = "trace")]
     pub fn read_discriminant(
         &self,
-        op: &impl Readable<'tcx, M::Provenance>,
+        op: &impl Projectable<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx, VariantIdx> {
         let ty = op.layout().ty;
         trace!("read_discriminant_value {:#?}", op.layout());
diff --git a/compiler/rustc_const_eval/src/interpret/intern.rs b/compiler/rustc_const_eval/src/interpret/intern.rs
index 8b0a2afa4d6..5df989b4c1d 100644
--- a/compiler/rustc_const_eval/src/interpret/intern.rs
+++ b/compiler/rustc_const_eval/src/interpret/intern.rs
@@ -20,6 +20,7 @@ use rustc_hir as hir;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
 use rustc_middle::mir::interpret::{ConstAllocation, CtfeProvenance, InterpResult};
 use rustc_middle::query::TyCtxtAt;
+use rustc_middle::span_bug;
 use rustc_middle::ty::layout::TyAndLayout;
 use rustc_span::def_id::LocalDefId;
 use rustc_span::sym;
@@ -223,37 +224,52 @@ pub fn intern_const_alloc_recursive<'tcx, M: CompileTimeMachine<'tcx, const_eval
             continue;
         }
 
-        // Crucially, we check this *before* checking whether the `alloc_id`
-        // has already been interned. The point of this check is to ensure that when
-        // there are multiple pointers to the same allocation, they are *all* immutable.
-        // Therefore it would be bad if we only checked the first pointer to any given
-        // allocation.
+        // Ensure that this is derived from a shared reference. Crucially, we check this *before*
+        // checking whether the `alloc_id` has already been interned. The point of this check is to
+        // ensure that when there are multiple pointers to the same allocation, they are *all*
+        // derived from a shared reference. Therefore it would be bad if we only checked the first
+        // pointer to any given allocation.
         // (It is likely not possible to actually have multiple pointers to the same allocation,
         // so alternatively we could also check that and ICE if there are multiple such pointers.)
+        // See <https://github.com/rust-lang/rust/pull/128543> for why we are checking for "shared
+        // reference" and not "immutable", i.e., for why we are allowing interior-mutable shared
+        // references: they can actually be created in safe code while pointing to apparently
+        // "immutable" values, via promotion or tail expression lifetime extension of
+        // `&None::<Cell<T>>`.
+        // We also exclude promoteds from this as `&mut []` can be promoted, which is a mutable
+        // reference pointing to an immutable (zero-sized) allocation. We rely on the promotion
+        // analysis not screwing up to ensure that it is sound to intern promoteds as immutable.
         if intern_kind != InternKind::Promoted
             && inner_mutability == Mutability::Not
-            && !prov.immutable()
+            && !prov.shared_ref()
         {
-            if ecx.tcx.try_get_global_alloc(alloc_id).is_some()
-                && !just_interned.contains(&alloc_id)
-            {
+            let is_already_global = ecx.tcx.try_get_global_alloc(alloc_id).is_some();
+            if is_already_global && !just_interned.contains(&alloc_id) {
                 // This is a pointer to some memory from another constant. We encounter mutable
                 // pointers to such memory since we do not always track immutability through
                 // these "global" pointers. Allowing them is harmless; the point of these checks
                 // during interning is to justify why we intern the *new* allocations immutably,
-                // so we can completely ignore existing allocations. We also don't need to add
-                // this to the todo list, since after all it is already interned.
+                // so we can completely ignore existing allocations.
+                // We can also skip the rest of this loop iteration, since after all it is already
+                // interned.
                 continue;
             }
-            // Found a mutable pointer inside a const where inner allocations should be
-            // immutable. We exclude promoteds from this, since things like `&mut []` and
-            // `&None::<Cell<i32>>` lead to promotion that can produce mutable pointers. We rely
-            // on the promotion analysis not screwing up to ensure that it is sound to intern
-            // promoteds as immutable.
-            trace!("found bad mutable pointer");
-            // Prefer dangling pointer errors over mutable pointer errors
-            if result.is_ok() {
-                result = Err(InternResult::FoundBadMutablePointer);
+            // If this is a dangling pointer, that's actually fine -- the problematic case is
+            // when there is memory there that someone might expect to be mutable, but we make it immutable.
+            let dangling = !is_already_global && !ecx.memory.alloc_map.contains_key(&alloc_id);
+            if !dangling {
+                // Found a mutable reference inside a const where inner allocations should be
+                // immutable.
+                if !ecx.tcx.sess.opts.unstable_opts.unleash_the_miri_inside_of_you {
+                    span_bug!(
+                        ecx.tcx.span,
+                        "the static const safety checks accepted mutable references they should not have accepted"
+                    );
+                }
+                // Prefer dangling pointer errors over mutable pointer errors
+                if result.is_ok() {
+                    result = Err(InternResult::FoundBadMutablePointer);
+                }
             }
         }
         if ecx.tcx.try_get_global_alloc(alloc_id).is_some() {
@@ -261,7 +277,6 @@ pub fn intern_const_alloc_recursive<'tcx, M: CompileTimeMachine<'tcx, const_eval
             debug_assert!(!ecx.memory.alloc_map.contains_key(&alloc_id));
             continue;
         }
-        just_interned.insert(alloc_id);
         // We always intern with `inner_mutability`, and furthermore we ensured above that if
         // that is "immutable", then there are *no* mutable pointers anywhere in the newly
         // interned memory -- justifying that we can indeed intern immutably. However this also
@@ -272,6 +287,7 @@ pub fn intern_const_alloc_recursive<'tcx, M: CompileTimeMachine<'tcx, const_eval
         // pointers before deciding which allocations can be made immutable; but for now we are
         // okay with losing some potential for immutability here. This can anyway only affect
         // `static mut`.
+        just_interned.insert(alloc_id);
         match intern_shallow(ecx, alloc_id, inner_mutability) {
             Ok(nested) => todo.extend(nested),
             Err(()) => {
diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs
index 6cfd7be48e6..8cab3c34eed 100644
--- a/compiler/rustc_const_eval/src/interpret/machine.rs
+++ b/compiler/rustc_const_eval/src/interpret/machine.rs
@@ -10,6 +10,7 @@ use rustc_apfloat::{Float, FloatConvert};
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_middle::query::TyCtxtAt;
 use rustc_middle::ty::layout::TyAndLayout;
+use rustc_middle::ty::Ty;
 use rustc_middle::{mir, ty};
 use rustc_span::def_id::DefId;
 use rustc_span::Span;
@@ -19,7 +20,7 @@ use rustc_target::spec::abi::Abi as CallAbi;
 use super::{
     throw_unsup, throw_unsup_format, AllocBytes, AllocId, AllocKind, AllocRange, Allocation,
     ConstAllocation, CtfeProvenance, FnArg, Frame, ImmTy, InterpCx, InterpResult, MPlaceTy,
-    MemoryKind, Misalignment, OpTy, PlaceTy, Pointer, Provenance, CTFE_ALLOC_SALT,
+    MemoryKind, Misalignment, OpTy, PlaceTy, Pointer, Provenance, RangeSet, CTFE_ALLOC_SALT,
 };
 
 /// Data returned by [`Machine::after_stack_pop`], and consumed by
@@ -578,6 +579,15 @@ pub trait Machine<'tcx>: Sized {
         ecx: &InterpCx<'tcx, Self>,
         instance: Option<ty::Instance<'tcx>>,
     ) -> usize;
+
+    fn cached_union_data_range<'e>(
+        _ecx: &'e mut InterpCx<'tcx, Self>,
+        _ty: Ty<'tcx>,
+        compute_range: impl FnOnce() -> RangeSet,
+    ) -> Cow<'e, RangeSet> {
+        // Default to no caching.
+        Cow::Owned(compute_range())
+    }
 }
 
 /// A lot of the flexibility above is just needed for `Miri`, but all "compile-time" machines
diff --git a/compiler/rustc_const_eval/src/interpret/memory.rs b/compiler/rustc_const_eval/src/interpret/memory.rs
index 45a5eb9bd52..d87588496c0 100644
--- a/compiler/rustc_const_eval/src/interpret/memory.rs
+++ b/compiler/rustc_const_eval/src/interpret/memory.rs
@@ -8,9 +8,8 @@
 
 use std::assert_matches::assert_matches;
 use std::borrow::Cow;
-use std::cell::Cell;
 use std::collections::VecDeque;
-use std::{fmt, ptr};
+use std::{fmt, mem, ptr};
 
 use rustc_ast::Mutability;
 use rustc_data_structures::fx::{FxHashSet, FxIndexMap};
@@ -118,7 +117,7 @@ pub struct Memory<'tcx, M: Machine<'tcx>> {
     /// This stores whether we are currently doing reads purely for the purpose of validation.
     /// Those reads do not trigger the machine's hooks for memory reads.
     /// Needless to say, this must only be set with great care!
-    validation_in_progress: Cell<bool>,
+    validation_in_progress: bool,
 }
 
 /// A reference to some allocation that was already bounds-checked for the given region
@@ -145,7 +144,7 @@ impl<'tcx, M: Machine<'tcx>> Memory<'tcx, M> {
             alloc_map: M::MemoryMap::default(),
             extra_fn_ptr_map: FxIndexMap::default(),
             dead_alloc_map: FxIndexMap::default(),
-            validation_in_progress: Cell::new(false),
+            validation_in_progress: false,
         }
     }
 
@@ -682,7 +681,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         // We want to call the hook on *all* accesses that involve an AllocId, including zero-sized
         // accesses. That means we cannot rely on the closure above or the `Some` branch below. We
         // do this after `check_and_deref_ptr` to ensure some basic sanity has already been checked.
-        if !self.memory.validation_in_progress.get() {
+        if !self.memory.validation_in_progress {
             if let Ok((alloc_id, ..)) = self.ptr_try_get_alloc_id(ptr, size_i64) {
                 M::before_alloc_read(self, alloc_id)?;
             }
@@ -690,7 +689,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
 
         if let Some((alloc_id, offset, prov, alloc)) = ptr_and_alloc {
             let range = alloc_range(offset, size);
-            if !self.memory.validation_in_progress.get() {
+            if !self.memory.validation_in_progress {
                 M::before_memory_read(
                     self.tcx,
                     &self.machine,
@@ -766,11 +765,14 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         let parts = self.get_ptr_access(ptr, size)?;
         if let Some((alloc_id, offset, prov)) = parts {
             let tcx = self.tcx;
+            let validation_in_progress = self.memory.validation_in_progress;
             // FIXME: can we somehow avoid looking up the allocation twice here?
             // We cannot call `get_raw_mut` inside `check_and_deref_ptr` as that would duplicate `&mut self`.
             let (alloc, machine) = self.get_alloc_raw_mut(alloc_id)?;
             let range = alloc_range(offset, size);
-            M::before_memory_write(tcx, machine, &mut alloc.extra, (alloc_id, prov), range)?;
+            if !validation_in_progress {
+                M::before_memory_write(tcx, machine, &mut alloc.extra, (alloc_id, prov), range)?;
+            }
             Ok(Some(AllocRefMut { alloc, range, tcx: *tcx, alloc_id }))
         } else {
             Ok(None)
@@ -1014,16 +1016,16 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     ///
     /// We do this so Miri's allocation access tracking does not show the validation
     /// reads as spurious accesses.
-    pub fn run_for_validation<R>(&self, f: impl FnOnce() -> R) -> R {
+    pub fn run_for_validation<R>(&mut self, f: impl FnOnce(&mut Self) -> R) -> R {
         // This deliberately uses `==` on `bool` to follow the pattern
         // `assert!(val.replace(new) == old)`.
         assert!(
-            self.memory.validation_in_progress.replace(true) == false,
+            mem::replace(&mut self.memory.validation_in_progress, true) == false,
             "`validation_in_progress` was already set"
         );
-        let res = f();
+        let res = f(self);
         assert!(
-            self.memory.validation_in_progress.replace(false) == true,
+            mem::replace(&mut self.memory.validation_in_progress, false) == true,
             "`validation_in_progress` was unset by someone else"
         );
         res
@@ -1115,6 +1117,10 @@ impl<'a, 'tcx, M: Machine<'tcx>> std::fmt::Debug for DumpAllocs<'a, 'tcx, M> {
 impl<'tcx, 'a, Prov: Provenance, Extra, Bytes: AllocBytes>
     AllocRefMut<'a, 'tcx, Prov, Extra, Bytes>
 {
+    pub fn as_ref<'b>(&'b self) -> AllocRef<'b, 'tcx, Prov, Extra, Bytes> {
+        AllocRef { alloc: self.alloc, range: self.range, tcx: self.tcx, alloc_id: self.alloc_id }
+    }
+
     /// `range` is relative to this allocation reference, not the base of the allocation.
     pub fn write_scalar(&mut self, range: AllocRange, val: Scalar<Prov>) -> InterpResult<'tcx> {
         let range = self.range.subrange(range);
@@ -1130,13 +1136,30 @@ impl<'tcx, 'a, Prov: Provenance, Extra, Bytes: AllocBytes>
         self.write_scalar(alloc_range(offset, self.tcx.data_layout().pointer_size), val)
     }
 
+    /// Mark the given sub-range (relative to this allocation reference) as uninitialized.
+    pub fn write_uninit(&mut self, range: AllocRange) -> InterpResult<'tcx> {
+        let range = self.range.subrange(range);
+        Ok(self
+            .alloc
+            .write_uninit(&self.tcx, range)
+            .map_err(|e| e.to_interp_error(self.alloc_id))?)
+    }
+
     /// Mark the entire referenced range as uninitialized
-    pub fn write_uninit(&mut self) -> InterpResult<'tcx> {
+    pub fn write_uninit_full(&mut self) -> InterpResult<'tcx> {
         Ok(self
             .alloc
             .write_uninit(&self.tcx, self.range)
             .map_err(|e| e.to_interp_error(self.alloc_id))?)
     }
+
+    /// Remove all provenance in the reference range.
+    pub fn clear_provenance(&mut self) -> InterpResult<'tcx> {
+        Ok(self
+            .alloc
+            .clear_provenance(&self.tcx, self.range)
+            .map_err(|e| e.to_interp_error(self.alloc_id))?)
+    }
 }
 
 impl<'tcx, 'a, Prov: Provenance, Extra, Bytes: AllocBytes> AllocRef<'a, 'tcx, Prov, Extra, Bytes> {
@@ -1278,7 +1301,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         };
         let src_alloc = self.get_alloc_raw(src_alloc_id)?;
         let src_range = alloc_range(src_offset, size);
-        assert!(!self.memory.validation_in_progress.get(), "we can't be copying during validation");
+        assert!(!self.memory.validation_in_progress, "we can't be copying during validation");
         M::before_memory_read(
             tcx,
             &self.machine,
diff --git a/compiler/rustc_const_eval/src/interpret/mod.rs b/compiler/rustc_const_eval/src/interpret/mod.rs
index 511756e3f86..561d681f804 100644
--- a/compiler/rustc_const_eval/src/interpret/mod.rs
+++ b/compiler/rustc_const_eval/src/interpret/mod.rs
@@ -33,11 +33,11 @@ pub(crate) use self::intrinsics::eval_nullary_intrinsic;
 pub use self::machine::{compile_time_machine, AllocMap, Machine, MayLeak, ReturnAction};
 pub use self::memory::{AllocKind, AllocRef, AllocRefMut, FnVal, Memory, MemoryKind};
 use self::operand::Operand;
-pub use self::operand::{ImmTy, Immediate, OpTy, Readable};
+pub use self::operand::{ImmTy, Immediate, OpTy};
 pub use self::place::{MPlaceTy, MemPlaceMeta, PlaceTy, Writeable};
 use self::place::{MemPlace, Place};
 pub use self::projection::{OffsetMode, Projectable};
 pub use self::stack::{Frame, FrameInfo, LocalState, StackPopCleanup, StackPopInfo};
 pub(crate) use self::util::create_static_alloc;
-pub use self::validity::{CtfeValidationMode, RefTracking};
+pub use self::validity::{CtfeValidationMode, RangeSet, RefTracking};
 pub use self::visitor::ValueVisitor;
diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs
index 9a8ccaa7cc5..b906e3422db 100644
--- a/compiler/rustc_const_eval/src/interpret/operand.rs
+++ b/compiler/rustc_const_eval/src/interpret/operand.rs
@@ -111,6 +111,46 @@ impl<Prov: Provenance> Immediate<Prov> {
             Immediate::Uninit => bug!("Got uninit where a scalar or scalar pair was expected"),
         }
     }
+
+    /// Assert that this immediate is a valid value for the given ABI.
+    pub fn assert_matches_abi(self, abi: Abi, cx: &impl HasDataLayout) {
+        match (self, abi) {
+            (Immediate::Scalar(scalar), Abi::Scalar(s)) => {
+                assert_eq!(scalar.size(), s.size(cx));
+                if !matches!(s.primitive(), abi::Pointer(..)) {
+                    assert!(matches!(scalar, Scalar::Int(..)));
+                }
+            }
+            (Immediate::ScalarPair(a_val, b_val), Abi::ScalarPair(a, b)) => {
+                assert_eq!(a_val.size(), a.size(cx));
+                if !matches!(a.primitive(), abi::Pointer(..)) {
+                    assert!(matches!(a_val, Scalar::Int(..)));
+                }
+                assert_eq!(b_val.size(), b.size(cx));
+                if !matches!(b.primitive(), abi::Pointer(..)) {
+                    assert!(matches!(b_val, Scalar::Int(..)));
+                }
+            }
+            (Immediate::Uninit, _) => {}
+            _ => {
+                bug!("value {self:?} does not match ABI {abi:?})",)
+            }
+        }
+    }
+
+    pub fn clear_provenance<'tcx>(&mut self) -> InterpResult<'tcx> {
+        match self {
+            Immediate::Scalar(s) => {
+                s.clear_provenance()?;
+            }
+            Immediate::ScalarPair(a, b) => {
+                a.clear_provenance()?;
+                b.clear_provenance()?;
+            }
+            Immediate::Uninit => {}
+        }
+        Ok(())
+    }
 }
 
 // ScalarPair needs a type to interpret, so we often have an immediate and a type together
@@ -490,32 +530,6 @@ impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for OpTy<'tcx, Prov> {
     }
 }
 
-/// The `Readable` trait describes interpreter values that one can read from.
-pub trait Readable<'tcx, Prov: Provenance>: Projectable<'tcx, Prov> {
-    fn as_mplace_or_imm(&self) -> Either<MPlaceTy<'tcx, Prov>, ImmTy<'tcx, Prov>>;
-}
-
-impl<'tcx, Prov: Provenance> Readable<'tcx, Prov> for OpTy<'tcx, Prov> {
-    #[inline(always)]
-    fn as_mplace_or_imm(&self) -> Either<MPlaceTy<'tcx, Prov>, ImmTy<'tcx, Prov>> {
-        self.as_mplace_or_imm()
-    }
-}
-
-impl<'tcx, Prov: Provenance> Readable<'tcx, Prov> for MPlaceTy<'tcx, Prov> {
-    #[inline(always)]
-    fn as_mplace_or_imm(&self) -> Either<MPlaceTy<'tcx, Prov>, ImmTy<'tcx, Prov>> {
-        Left(self.clone())
-    }
-}
-
-impl<'tcx, Prov: Provenance> Readable<'tcx, Prov> for ImmTy<'tcx, Prov> {
-    #[inline(always)]
-    fn as_mplace_or_imm(&self) -> Either<MPlaceTy<'tcx, Prov>, ImmTy<'tcx, Prov>> {
-        Right(self.clone())
-    }
-}
-
 impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     /// Try reading an immediate in memory; this is interesting particularly for `ScalarPair`.
     /// Returns `None` if the layout does not permit loading this as a value.
@@ -588,9 +602,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     /// ConstProp needs it, though.
     pub fn read_immediate_raw(
         &self,
-        src: &impl Readable<'tcx, M::Provenance>,
+        src: &impl Projectable<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx, Either<MPlaceTy<'tcx, M::Provenance>, ImmTy<'tcx, M::Provenance>>> {
-        Ok(match src.as_mplace_or_imm() {
+        Ok(match src.to_op(self)?.as_mplace_or_imm() {
             Left(ref mplace) => {
                 if let Some(val) = self.read_immediate_from_mplace_raw(mplace)? {
                     Right(val)
@@ -608,7 +622,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     #[inline(always)]
     pub fn read_immediate(
         &self,
-        op: &impl Readable<'tcx, M::Provenance>,
+        op: &impl Projectable<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> {
         if !matches!(
             op.layout().abi,
@@ -627,7 +641,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     /// Read a scalar from a place
     pub fn read_scalar(
         &self,
-        op: &impl Readable<'tcx, M::Provenance>,
+        op: &impl Projectable<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
         Ok(self.read_immediate(op)?.to_scalar())
     }
@@ -638,21 +652,21 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     /// Read a pointer from a place.
     pub fn read_pointer(
         &self,
-        op: &impl Readable<'tcx, M::Provenance>,
+        op: &impl Projectable<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx, Pointer<Option<M::Provenance>>> {
         self.read_scalar(op)?.to_pointer(self)
     }
     /// Read a pointer-sized unsigned integer from a place.
     pub fn read_target_usize(
         &self,
-        op: &impl Readable<'tcx, M::Provenance>,
+        op: &impl Projectable<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx, u64> {
         self.read_scalar(op)?.to_target_usize(self)
     }
     /// Read a pointer-sized signed integer from a place.
     pub fn read_target_isize(
         &self,
-        op: &impl Readable<'tcx, M::Provenance>,
+        op: &impl Projectable<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx, i64> {
         self.read_scalar(op)?.to_target_isize(self)
     }
@@ -717,7 +731,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
         match place.as_mplace_or_local() {
             Left(mplace) => Ok(mplace.into()),
-            Right((local, offset, locals_addr)) => {
+            Right((local, offset, locals_addr, _)) => {
                 debug_assert!(place.layout.is_sized()); // only sized locals can ever be `Place::Local`.
                 debug_assert_eq!(locals_addr, self.frame().locals_addr());
                 let base = self.local_to_op(local, None)?;
diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs
index 840f7986c6e..3b14142da02 100644
--- a/compiler/rustc_const_eval/src/interpret/place.rs
+++ b/compiler/rustc_const_eval/src/interpret/place.rs
@@ -15,7 +15,7 @@ use tracing::{instrument, trace};
 use super::{
     alloc_range, mir_assign_valid_types, AllocRef, AllocRefMut, CheckAlignMsg, CtfeProvenance,
     ImmTy, Immediate, InterpCx, InterpResult, Machine, MemoryKind, Misalignment, OffsetMode, OpTy,
-    Operand, Pointer, Projectable, Provenance, Readable, Scalar,
+    Operand, Pointer, Projectable, Provenance, Scalar,
 };
 
 #[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)]
@@ -180,7 +180,8 @@ pub(super) enum Place<Prov: Provenance = CtfeProvenance> {
     Ptr(MemPlace<Prov>),
 
     /// To support alloc-free locals, we are able to write directly to a local. The offset indicates
-    /// where in the local this place is located; if it is `None`, no projection has been applied.
+    /// where in the local this place is located; if it is `None`, no projection has been applied
+    /// and the type of the place is exactly the type of the local.
     /// Such projections are meaningful even if the offset is 0, since they can change layouts.
     /// (Without that optimization, we'd just always be a `MemPlace`.)
     /// `Local` places always refer to the current stack frame, so they are unstable under
@@ -231,10 +232,12 @@ impl<'tcx, Prov: Provenance> PlaceTy<'tcx, Prov> {
     #[inline(always)]
     pub fn as_mplace_or_local(
         &self,
-    ) -> Either<MPlaceTy<'tcx, Prov>, (mir::Local, Option<Size>, usize)> {
+    ) -> Either<MPlaceTy<'tcx, Prov>, (mir::Local, Option<Size>, usize, TyAndLayout<'tcx>)> {
         match self.place {
             Place::Ptr(mplace) => Left(MPlaceTy { mplace, layout: self.layout }),
-            Place::Local { local, offset, locals_addr } => Right((local, offset, locals_addr)),
+            Place::Local { local, offset, locals_addr } => {
+                Right((local, offset, locals_addr, self.layout))
+            }
         }
     }
 
@@ -277,7 +280,7 @@ impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for PlaceTy<'tcx, Prov> {
     ) -> InterpResult<'tcx, Self> {
         Ok(match self.as_mplace_or_local() {
             Left(mplace) => mplace.offset_with_meta(offset, mode, meta, layout, ecx)?.into(),
-            Right((local, old_offset, locals_addr)) => {
+            Right((local, old_offset, locals_addr, _)) => {
                 debug_assert!(layout.is_sized(), "unsized locals should live in memory");
                 assert_matches!(meta, MemPlaceMeta::None); // we couldn't store it anyway...
                 // `Place::Local` are always in-bounds of their surrounding local, so we can just
@@ -328,9 +331,7 @@ impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
 
 /// The `Weiteable` trait describes interpreter values that can be written to.
 pub trait Writeable<'tcx, Prov: Provenance>: Projectable<'tcx, Prov> {
-    fn as_mplace_or_local(
-        &self,
-    ) -> Either<MPlaceTy<'tcx, Prov>, (mir::Local, Option<Size>, usize, TyAndLayout<'tcx>)>;
+    fn to_place(&self) -> PlaceTy<'tcx, Prov>;
 
     fn force_mplace<M: Machine<'tcx, Provenance = Prov>>(
         &self,
@@ -340,11 +341,8 @@ pub trait Writeable<'tcx, Prov: Provenance>: Projectable<'tcx, Prov> {
 
 impl<'tcx, Prov: Provenance> Writeable<'tcx, Prov> for PlaceTy<'tcx, Prov> {
     #[inline(always)]
-    fn as_mplace_or_local(
-        &self,
-    ) -> Either<MPlaceTy<'tcx, Prov>, (mir::Local, Option<Size>, usize, TyAndLayout<'tcx>)> {
-        self.as_mplace_or_local()
-            .map_right(|(local, offset, locals_addr)| (local, offset, locals_addr, self.layout))
+    fn to_place(&self) -> PlaceTy<'tcx, Prov> {
+        self.clone()
     }
 
     #[inline(always)]
@@ -358,10 +356,8 @@ impl<'tcx, Prov: Provenance> Writeable<'tcx, Prov> for PlaceTy<'tcx, Prov> {
 
 impl<'tcx, Prov: Provenance> Writeable<'tcx, Prov> for MPlaceTy<'tcx, Prov> {
     #[inline(always)]
-    fn as_mplace_or_local(
-        &self,
-    ) -> Either<MPlaceTy<'tcx, Prov>, (mir::Local, Option<Size>, usize, TyAndLayout<'tcx>)> {
-        Left(self.clone())
+    fn to_place(&self) -> PlaceTy<'tcx, Prov> {
+        self.clone().into()
     }
 
     #[inline(always)]
@@ -436,7 +432,7 @@ where
     #[instrument(skip(self), level = "trace")]
     pub fn deref_pointer(
         &self,
-        src: &impl Readable<'tcx, M::Provenance>,
+        src: &impl Projectable<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
         if src.layout().ty.is_box() {
             // Derefer should have removed all Box derefs.
@@ -562,6 +558,40 @@ where
         Ok(place)
     }
 
+    /// Given a place, returns either the underlying mplace or a reference to where the value of
+    /// this place is stored.
+    fn as_mplace_or_mutable_local(
+        &mut self,
+        place: &PlaceTy<'tcx, M::Provenance>,
+    ) -> InterpResult<
+        'tcx,
+        Either<MPlaceTy<'tcx, M::Provenance>, (&mut Immediate<M::Provenance>, TyAndLayout<'tcx>)>,
+    > {
+        Ok(match place.to_place().as_mplace_or_local() {
+            Left(mplace) => Left(mplace),
+            Right((local, offset, locals_addr, layout)) => {
+                if offset.is_some() {
+                    // This has been projected to a part of this local, or had the type changed.
+                    // FIMXE: there are cases where we could still avoid allocating an mplace.
+                    Left(place.force_mplace(self)?)
+                } else {
+                    debug_assert_eq!(locals_addr, self.frame().locals_addr());
+                    debug_assert_eq!(self.layout_of_local(self.frame(), local, None)?, layout);
+                    match self.frame_mut().locals[local].access_mut()? {
+                        Operand::Indirect(mplace) => {
+                            // The local is in memory.
+                            Left(MPlaceTy { mplace: *mplace, layout })
+                        }
+                        Operand::Immediate(local_val) => {
+                            // The local still has the optimized representation.
+                            Right((local_val, layout))
+                        }
+                    }
+                }
+            }
+        })
+    }
+
     /// Write an immediate to a place
     #[inline(always)]
     #[instrument(skip(self), level = "trace")]
@@ -574,9 +604,11 @@ where
 
         if M::enforce_validity(self, dest.layout()) {
             // Data got changed, better make sure it matches the type!
+            // Also needed to reset padding.
             self.validate_operand(
-                &dest.to_op(self)?,
+                &dest.to_place(),
                 M::enforce_validity_recursively(self, dest.layout()),
+                /*reset_provenance_and_padding*/ true,
             )?;
         }
 
@@ -606,67 +638,27 @@ where
     /// Write an immediate to a place.
     /// If you use this you are responsible for validating that things got copied at the
     /// right type.
-    fn write_immediate_no_validate(
+    pub(super) fn write_immediate_no_validate(
         &mut self,
         src: Immediate<M::Provenance>,
         dest: &impl Writeable<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx> {
         assert!(dest.layout().is_sized(), "Cannot write unsized immediate data");
 
-        // See if we can avoid an allocation. This is the counterpart to `read_immediate_raw`,
-        // but not factored as a separate function.
-        let mplace = match dest.as_mplace_or_local() {
-            Right((local, offset, locals_addr, layout)) => {
-                if offset.is_some() {
-                    // This has been projected to a part of this local. We could have complicated
-                    // logic to still keep this local as an `Operand`... but it's much easier to
-                    // just fall back to the indirect path.
-                    dest.force_mplace(self)?
-                } else {
-                    debug_assert_eq!(locals_addr, self.frame().locals_addr());
-                    match self.frame_mut().locals[local].access_mut()? {
-                        Operand::Immediate(local_val) => {
-                            // Local can be updated in-place.
-                            *local_val = src;
-                            // Double-check that the value we are storing and the local fit to each other.
-                            // (*After* doing the update for borrow checker reasons.)
-                            if cfg!(debug_assertions) {
-                                let local_layout =
-                                    self.layout_of_local(&self.frame(), local, None)?;
-                                match (src, local_layout.abi) {
-                                    (Immediate::Scalar(scalar), Abi::Scalar(s)) => {
-                                        assert_eq!(scalar.size(), s.size(self))
-                                    }
-                                    (
-                                        Immediate::ScalarPair(a_val, b_val),
-                                        Abi::ScalarPair(a, b),
-                                    ) => {
-                                        assert_eq!(a_val.size(), a.size(self));
-                                        assert_eq!(b_val.size(), b.size(self));
-                                    }
-                                    (Immediate::Uninit, _) => {}
-                                    (src, abi) => {
-                                        bug!(
-                                            "value {src:?} cannot be written into local with type {} (ABI {abi:?})",
-                                            local_layout.ty
-                                        )
-                                    }
-                                };
-                            }
-                            return Ok(());
-                        }
-                        Operand::Indirect(mplace) => {
-                            // The local is in memory, go on below.
-                            MPlaceTy { mplace: *mplace, layout }
-                        }
-                    }
+        match self.as_mplace_or_mutable_local(&dest.to_place())? {
+            Right((local_val, local_layout)) => {
+                // Local can be updated in-place.
+                *local_val = src;
+                // Double-check that the value we are storing and the local fit to each other.
+                if cfg!(debug_assertions) {
+                    src.assert_matches_abi(local_layout.abi, self);
                 }
             }
-            Left(mplace) => mplace, // already referring to memory
-        };
-
-        // This is already in memory, write there.
-        self.write_immediate_to_mplace_no_validate(src, mplace.layout, mplace.mplace)
+            Left(mplace) => {
+                self.write_immediate_to_mplace_no_validate(src, mplace.layout, mplace.mplace)?;
+            }
+        }
+        Ok(())
     }
 
     /// Write an immediate to memory.
@@ -678,6 +670,9 @@ where
         layout: TyAndLayout<'tcx>,
         dest: MemPlace<M::Provenance>,
     ) -> InterpResult<'tcx> {
+        if cfg!(debug_assertions) {
+            value.assert_matches_abi(layout.abi, self);
+        }
         // Note that it is really important that the type here is the right one, and matches the
         // type things are read at. In case `value` is a `ScalarPair`, we don't do any magic here
         // to handle padding properly, which is only correct if we never look at this data with the
@@ -691,15 +686,7 @@ where
 
         match value {
             Immediate::Scalar(scalar) => {
-                let Abi::Scalar(s) = layout.abi else {
-                    span_bug!(
-                        self.cur_span(),
-                        "write_immediate_to_mplace: invalid Scalar layout: {layout:#?}",
-                    )
-                };
-                let size = s.size(&tcx);
-                assert_eq!(size, layout.size, "abi::Scalar size does not match layout size");
-                alloc.write_scalar(alloc_range(Size::ZERO, size), scalar)
+                alloc.write_scalar(alloc_range(Size::ZERO, scalar.size()), scalar)
             }
             Immediate::ScalarPair(a_val, b_val) => {
                 let Abi::ScalarPair(a, b) = layout.abi else {
@@ -709,18 +696,19 @@ where
                         layout
                     )
                 };
-                let (a_size, b_size) = (a.size(&tcx), b.size(&tcx));
-                let b_offset = a_size.align_to(b.align(&tcx).abi);
+                let b_offset = a.size(&tcx).align_to(b.align(&tcx).abi);
                 assert!(b_offset.bytes() > 0); // in `operand_field` we use the offset to tell apart the fields
 
                 // It is tempting to verify `b_offset` against `layout.fields.offset(1)`,
                 // but that does not work: We could be a newtype around a pair, then the
                 // fields do not match the `ScalarPair` components.
 
-                alloc.write_scalar(alloc_range(Size::ZERO, a_size), a_val)?;
-                alloc.write_scalar(alloc_range(b_offset, b_size), b_val)
+                alloc.write_scalar(alloc_range(Size::ZERO, a_val.size()), a_val)?;
+                alloc.write_scalar(alloc_range(b_offset, b_val.size()), b_val)?;
+                // We don't have to reset padding here, `write_immediate` will anyway do a validation run.
+                Ok(())
             }
-            Immediate::Uninit => alloc.write_uninit(),
+            Immediate::Uninit => alloc.write_uninit_full(),
         }
     }
 
@@ -728,35 +716,38 @@ where
         &mut self,
         dest: &impl Writeable<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx> {
-        let mplace = match dest.as_mplace_or_local() {
-            Left(mplace) => mplace,
-            Right((local, offset, locals_addr, layout)) => {
-                if offset.is_some() {
-                    // This has been projected to a part of this local. We could have complicated
-                    // logic to still keep this local as an `Operand`... but it's much easier to
-                    // just fall back to the indirect path.
-                    // FIXME: share the logic with `write_immediate_no_validate`.
-                    dest.force_mplace(self)?
-                } else {
-                    debug_assert_eq!(locals_addr, self.frame().locals_addr());
-                    match self.frame_mut().locals[local].access_mut()? {
-                        Operand::Immediate(local) => {
-                            *local = Immediate::Uninit;
-                            return Ok(());
-                        }
-                        Operand::Indirect(mplace) => {
-                            // The local is in memory, go on below.
-                            MPlaceTy { mplace: *mplace, layout }
-                        }
-                    }
-                }
+        match self.as_mplace_or_mutable_local(&dest.to_place())? {
+            Right((local_val, _local_layout)) => {
+                *local_val = Immediate::Uninit;
             }
-        };
-        let Some(mut alloc) = self.get_place_alloc_mut(&mplace)? else {
-            // Zero-sized access
-            return Ok(());
-        };
-        alloc.write_uninit()?;
+            Left(mplace) => {
+                let Some(mut alloc) = self.get_place_alloc_mut(&mplace)? else {
+                    // Zero-sized access
+                    return Ok(());
+                };
+                alloc.write_uninit_full()?;
+            }
+        }
+        Ok(())
+    }
+
+    /// Remove all provenance in the given place.
+    pub fn clear_provenance(
+        &mut self,
+        dest: &impl Writeable<'tcx, M::Provenance>,
+    ) -> InterpResult<'tcx> {
+        match self.as_mplace_or_mutable_local(&dest.to_place())? {
+            Right((local_val, _local_layout)) => {
+                local_val.clear_provenance()?;
+            }
+            Left(mplace) => {
+                let Some(mut alloc) = self.get_place_alloc_mut(&mplace)? else {
+                    // Zero-sized access
+                    return Ok(());
+                };
+                alloc.clear_provenance()?;
+            }
+        }
         Ok(())
     }
 
@@ -768,7 +759,7 @@ where
     #[inline(always)]
     pub(super) fn copy_op_no_dest_validation(
         &mut self,
-        src: &impl Readable<'tcx, M::Provenance>,
+        src: &impl Projectable<'tcx, M::Provenance>,
         dest: &impl Writeable<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx> {
         self.copy_op_inner(
@@ -781,7 +772,7 @@ where
     #[inline(always)]
     pub fn copy_op_allow_transmute(
         &mut self,
-        src: &impl Readable<'tcx, M::Provenance>,
+        src: &impl Projectable<'tcx, M::Provenance>,
         dest: &impl Writeable<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx> {
         self.copy_op_inner(
@@ -794,7 +785,7 @@ where
     #[inline(always)]
     pub fn copy_op(
         &mut self,
-        src: &impl Readable<'tcx, M::Provenance>,
+        src: &impl Projectable<'tcx, M::Provenance>,
         dest: &impl Writeable<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx> {
         self.copy_op_inner(
@@ -808,28 +799,35 @@ where
     #[instrument(skip(self), level = "trace")]
     fn copy_op_inner(
         &mut self,
-        src: &impl Readable<'tcx, M::Provenance>,
+        src: &impl Projectable<'tcx, M::Provenance>,
         dest: &impl Writeable<'tcx, M::Provenance>,
         allow_transmute: bool,
         validate_dest: bool,
     ) -> InterpResult<'tcx> {
-        // Generally for transmutation, data must be valid both at the old and new type.
-        // But if the types are the same, the 2nd validation below suffices.
-        if src.layout().ty != dest.layout().ty && M::enforce_validity(self, src.layout()) {
-            self.validate_operand(
-                &src.to_op(self)?,
-                M::enforce_validity_recursively(self, src.layout()),
-            )?;
-        }
+        // These are technically *two* typed copies: `src` is a not-yet-loaded value,
+        // so we're going a typed copy at `src` type from there to some intermediate storage.
+        // And then we're doing a second typed copy from that intermediate storage to `dest`.
+        // But as an optimization, we only make a single direct copy here.
 
         // Do the actual copy.
         self.copy_op_no_validate(src, dest, allow_transmute)?;
 
         if validate_dest && M::enforce_validity(self, dest.layout()) {
-            // Data got changed, better make sure it matches the type!
+            let dest = dest.to_place();
+            // Given that there were two typed copies, we have to ensure this is valid at both types,
+            // and we have to ensure this loses provenance and padding according to both types.
+            // But if the types are identical, we only do one pass.
+            if allow_transmute && src.layout().ty != dest.layout().ty {
+                self.validate_operand(
+                    &dest.transmute(src.layout(), self)?,
+                    M::enforce_validity_recursively(self, src.layout()),
+                    /*reset_provenance_and_padding*/ true,
+                )?;
+            }
             self.validate_operand(
-                &dest.to_op(self)?,
+                &dest,
                 M::enforce_validity_recursively(self, dest.layout()),
+                /*reset_provenance_and_padding*/ true,
             )?;
         }
 
@@ -843,7 +841,7 @@ where
     #[instrument(skip(self), level = "trace")]
     fn copy_op_no_validate(
         &mut self,
-        src: &impl Readable<'tcx, M::Provenance>,
+        src: &impl Projectable<'tcx, M::Provenance>,
         dest: &impl Writeable<'tcx, M::Provenance>,
         allow_transmute: bool,
     ) -> InterpResult<'tcx> {
diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs
index 26b7251f6db..469af35ec1b 100644
--- a/compiler/rustc_const_eval/src/interpret/validity.rs
+++ b/compiler/rustc_const_eval/src/interpret/validity.rs
@@ -4,6 +4,7 @@
 //! That's useful because it means other passes (e.g. promotion) can rely on `const`s
 //! to be const-safe.
 
+use std::borrow::Cow;
 use std::fmt::Write;
 use std::hash::Hash;
 use std::num::NonZero;
@@ -13,25 +14,25 @@ use hir::def::DefKind;
 use rustc_ast::Mutability;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_hir as hir;
-use rustc_middle::bug;
 use rustc_middle::mir::interpret::ValidationErrorKind::{self, *};
 use rustc_middle::mir::interpret::{
-    ExpectedKind, InterpError, InvalidMetaKind, Misalignment, PointerKind, Provenance,
+    alloc_range, ExpectedKind, InterpError, InvalidMetaKind, Misalignment, PointerKind, Provenance,
     UnsupportedOpInfo, ValidationErrorInfo,
 };
-use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
-use rustc_middle::ty::{self, Ty};
+use rustc_middle::ty::layout::{LayoutCx, LayoutOf, TyAndLayout};
+use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_middle::{bug, span_bug};
 use rustc_span::symbol::{sym, Symbol};
 use rustc_target::abi::{
-    Abi, FieldIdx, Scalar as ScalarAbi, Size, VariantIdx, Variants, WrappingRange,
+    Abi, FieldIdx, FieldsShape, Scalar as ScalarAbi, Size, VariantIdx, Variants, WrappingRange,
 };
 use tracing::trace;
 
 use super::machine::AllocMap;
 use super::{
     err_ub, format_interp_error, throw_ub, AllocId, AllocKind, CheckInAllocMsg, GlobalAlloc, ImmTy,
-    Immediate, InterpCx, InterpResult, MPlaceTy, Machine, MemPlaceMeta, OpTy, Pointer, Projectable,
-    Scalar, ValueVisitor,
+    Immediate, InterpCx, InterpResult, MPlaceTy, Machine, MemPlaceMeta, PlaceTy, Pointer,
+    Projectable, Scalar, ValueVisitor,
 };
 
 // for the validation errors
@@ -125,6 +126,7 @@ pub enum PathElem {
     EnumTag,
     CoroutineTag,
     DynDowncast,
+    Vtable,
 }
 
 /// Extra things to check for during validation of CTFE results.
@@ -163,22 +165,22 @@ impl<T: Clone + Eq + Hash + std::fmt::Debug, PATH: Default> RefTracking<T, PATH>
     pub fn empty() -> Self {
         RefTracking { seen: FxHashSet::default(), todo: vec![] }
     }
-    pub fn new(op: T) -> Self {
+    pub fn new(val: T) -> Self {
         let mut ref_tracking_for_consts =
-            RefTracking { seen: FxHashSet::default(), todo: vec![(op.clone(), PATH::default())] };
-        ref_tracking_for_consts.seen.insert(op);
+            RefTracking { seen: FxHashSet::default(), todo: vec![(val.clone(), PATH::default())] };
+        ref_tracking_for_consts.seen.insert(val);
         ref_tracking_for_consts
     }
     pub fn next(&mut self) -> Option<(T, PATH)> {
         self.todo.pop()
     }
 
-    fn track(&mut self, op: T, path: impl FnOnce() -> PATH) {
-        if self.seen.insert(op.clone()) {
-            trace!("Recursing below ptr {:#?}", op);
+    fn track(&mut self, val: T, path: impl FnOnce() -> PATH) {
+        if self.seen.insert(val.clone()) {
+            trace!("Recursing below ptr {:#?}", val);
             let path = path();
             // Remember to come back to this later.
-            self.todo.push((op, path));
+            self.todo.push((val, path));
         }
     }
 }
@@ -204,11 +206,62 @@ fn write_path(out: &mut String, path: &[PathElem]) {
             // not the root.
             Deref => write!(out, ".<deref>"),
             DynDowncast => write!(out, ".<dyn-downcast>"),
+            Vtable => write!(out, ".<vtable>"),
         }
         .unwrap()
     }
 }
 
+/// Represents a set of `Size` values as a sorted list of ranges.
+// These are (offset, length) pairs, and they are sorted and mutually disjoint,
+// and never adjacent (i.e. there's always a gap between two of them).
+#[derive(Debug, Clone)]
+pub struct RangeSet(Vec<(Size, Size)>);
+
+impl RangeSet {
+    fn add_range(&mut self, offset: Size, size: Size) {
+        if size.bytes() == 0 {
+            // No need to track empty ranges.
+            return;
+        }
+        let v = &mut self.0;
+        // We scan for a partition point where the left partition is all the elements that end
+        // strictly before we start. Those are elements that are too "low" to merge with us.
+        let idx =
+            v.partition_point(|&(other_offset, other_size)| other_offset + other_size < offset);
+        // Now we want to either merge with the first element of the second partition, or insert ourselves before that.
+        if let Some(&(other_offset, other_size)) = v.get(idx)
+            && offset + size >= other_offset
+        {
+            // Their end is >= our start (otherwise it would not be in the 2nd partition) and
+            // our end is >= their start. This means we can merge the ranges.
+            let new_start = other_offset.min(offset);
+            let mut new_end = (other_offset + other_size).max(offset + size);
+            // We grew to the right, so merge with overlapping/adjacent elements.
+            // (We also may have grown to the left, but that can never make us adjacent with
+            // anything there since we selected the first such candidate via `partition_point`.)
+            let mut scan_right = 1;
+            while let Some(&(next_offset, next_size)) = v.get(idx + scan_right)
+                && new_end >= next_offset
+            {
+                // Increase our size to absorb the next element.
+                new_end = new_end.max(next_offset + next_size);
+                // Look at the next element.
+                scan_right += 1;
+            }
+            // Update the element we grew.
+            v[idx] = (new_start, new_end - new_start);
+            // Remove the elements we absorbed (if any).
+            if scan_right > 1 {
+                drop(v.drain((idx + 1)..(idx + scan_right)));
+            }
+        } else {
+            // Insert new element.
+            v.insert(idx, (offset, size));
+        }
+    }
+}
+
 struct ValidityVisitor<'rt, 'tcx, M: Machine<'tcx>> {
     /// The `path` may be pushed to, but the part that is present when a function
     /// starts must not be changed!  `visit_fields` and `visit_array` rely on
@@ -217,7 +270,17 @@ struct ValidityVisitor<'rt, 'tcx, M: Machine<'tcx>> {
     ref_tracking: Option<&'rt mut RefTracking<MPlaceTy<'tcx, M::Provenance>, Vec<PathElem>>>,
     /// `None` indicates this is not validating for CTFE (but for runtime).
     ctfe_mode: Option<CtfeValidationMode>,
-    ecx: &'rt InterpCx<'tcx, M>,
+    ecx: &'rt mut InterpCx<'tcx, M>,
+    /// Whether provenance should be reset outside of pointers (emulating the effect of a typed
+    /// copy).
+    reset_provenance_and_padding: bool,
+    /// This tracks which byte ranges in this value contain data; the remaining bytes are padding.
+    /// The ideal representation here would be pointer-length pairs, but to keep things more compact
+    /// we only store a (range) set of offsets -- the base pointer is the same throughout the entire
+    /// visit, after all.
+    /// If this is `Some`, then `reset_provenance_and_padding` must be true (but not vice versa:
+    /// we might not track data vs padding bytes if the operand isn't stored in memory anyway).
+    data_bytes: Option<RangeSet>,
 }
 
 impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
@@ -287,8 +350,14 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
             // arrays/slices
             ty::Array(..) | ty::Slice(..) => PathElem::ArrayElem(field),
 
+            // dyn* vtables
+            ty::Dynamic(_, _, ty::DynKind::DynStar) if field == 1 => PathElem::Vtable,
+
             // dyn traits
-            ty::Dynamic(..) => PathElem::DynDowncast,
+            ty::Dynamic(..) => {
+                assert_eq!(field, 0);
+                PathElem::DynDowncast
+            }
 
             // nothing else has an aggregate layout
             _ => bug!("aggregate_field_path_elem: got non-aggregate type {:?}", layout.ty),
@@ -314,11 +383,11 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
 
     fn read_immediate(
         &self,
-        op: &OpTy<'tcx, M::Provenance>,
+        val: &PlaceTy<'tcx, M::Provenance>,
         expected: ExpectedKind,
     ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> {
         Ok(try_validation!(
-            self.ecx.read_immediate(op),
+            self.ecx.read_immediate(val),
             self.path,
             Ub(InvalidUninitBytes(None)) =>
                 Uninit { expected },
@@ -332,10 +401,40 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
 
     fn read_scalar(
         &self,
-        op: &OpTy<'tcx, M::Provenance>,
+        val: &PlaceTy<'tcx, M::Provenance>,
         expected: ExpectedKind,
     ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
-        Ok(self.read_immediate(op, expected)?.to_scalar())
+        Ok(self.read_immediate(val, expected)?.to_scalar())
+    }
+
+    fn deref_pointer(
+        &mut self,
+        val: &PlaceTy<'tcx, M::Provenance>,
+        expected: ExpectedKind,
+    ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
+        // Not using `ecx.deref_pointer` since we want to use our `read_immediate` wrapper.
+        let imm = self.read_immediate(val, expected)?;
+        // Reset provenance: ensure slice tail metadata does not preserve provenance,
+        // and ensure all pointers do not preserve partial provenance.
+        if self.reset_provenance_and_padding {
+            if matches!(imm.layout.abi, Abi::Scalar(..)) {
+                // A thin pointer. If it has provenance, we don't have to do anything.
+                // If it does not, ensure we clear the provenance in memory.
+                if matches!(imm.to_scalar(), Scalar::Int(..)) {
+                    self.ecx.clear_provenance(val)?;
+                }
+            } else {
+                // A wide pointer. This means we have to worry both about the pointer itself and the
+                // metadata. We do the lazy thing and just write back the value we got. Just
+                // clearing provenance in a targeted manner would be more efficient, but unless this
+                // is a perf hotspot it's just not worth the effort.
+                self.ecx.write_immediate_no_validate(*imm, val)?;
+            }
+            // The entire thing is data, not padding.
+            self.add_data_range_place(val);
+        }
+        // Now turn it into a place.
+        self.ecx.ref_to_mplace(&imm)
     }
 
     fn check_wide_ptr_meta(
@@ -376,11 +475,10 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
     /// Check a reference or `Box`.
     fn check_safe_pointer(
         &mut self,
-        value: &OpTy<'tcx, M::Provenance>,
+        value: &PlaceTy<'tcx, M::Provenance>,
         ptr_kind: PointerKind,
     ) -> InterpResult<'tcx> {
-        // Not using `deref_pointer` since we want to use our `read_immediate` wrapper.
-        let place = self.ecx.ref_to_mplace(&self.read_immediate(value, ptr_kind.into())?)?;
+        let place = self.deref_pointer(value, ptr_kind.into())?;
         // Handle wide pointers.
         // Check metadata early, for better diagnostics
         if place.layout.is_unsized() {
@@ -519,6 +617,13 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
                         if ptr_expected_mutbl == Mutability::Mut
                             && alloc_actual_mutbl == Mutability::Not
                         {
+                            if !self.ecx.tcx.sess.opts.unstable_opts.unleash_the_miri_inside_of_you
+                            {
+                                span_bug!(
+                                    self.ecx.tcx.span,
+                                    "the static const safety checks accepted mutable references they should not have accepted"
+                                );
+                            }
                             throw_validation_failure!(self.path, MutableRefToImmutable);
                         }
                         // In a const, everything must be completely immutable.
@@ -564,31 +669,39 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
     /// Note that not all of these have `FieldsShape::Primitive`, e.g. wide references.
     fn try_visit_primitive(
         &mut self,
-        value: &OpTy<'tcx, M::Provenance>,
+        value: &PlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx, bool> {
         // Go over all the primitive types
         let ty = value.layout.ty;
         match ty.kind() {
             ty::Bool => {
-                let value = self.read_scalar(value, ExpectedKind::Bool)?;
+                let scalar = self.read_scalar(value, ExpectedKind::Bool)?;
                 try_validation!(
-                    value.to_bool(),
+                    scalar.to_bool(),
                     self.path,
                     Ub(InvalidBool(..)) => ValidationErrorKind::InvalidBool {
-                        value: format!("{value:x}"),
+                        value: format!("{scalar:x}"),
                     }
                 );
+                if self.reset_provenance_and_padding {
+                    self.ecx.clear_provenance(value)?;
+                    self.add_data_range_place(value);
+                }
                 Ok(true)
             }
             ty::Char => {
-                let value = self.read_scalar(value, ExpectedKind::Char)?;
+                let scalar = self.read_scalar(value, ExpectedKind::Char)?;
                 try_validation!(
-                    value.to_char(),
+                    scalar.to_char(),
                     self.path,
                     Ub(InvalidChar(..)) => ValidationErrorKind::InvalidChar {
-                        value: format!("{value:x}"),
+                        value: format!("{scalar:x}"),
                     }
                 );
+                if self.reset_provenance_and_padding {
+                    self.ecx.clear_provenance(value)?;
+                    self.add_data_range_place(value);
+                }
                 Ok(true)
             }
             ty::Float(_) | ty::Int(_) | ty::Uint(_) => {
@@ -602,11 +715,14 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
                         ExpectedKind::Int
                     },
                 )?;
+                if self.reset_provenance_and_padding {
+                    self.ecx.clear_provenance(value)?;
+                    self.add_data_range_place(value);
+                }
                 Ok(true)
             }
             ty::RawPtr(..) => {
-                let place =
-                    self.ecx.ref_to_mplace(&self.read_immediate(value, ExpectedKind::RawPtr)?)?;
+                let place = self.deref_pointer(value, ExpectedKind::RawPtr)?;
                 if place.layout.is_unsized() {
                     self.check_wide_ptr_meta(place.meta(), place.layout)?;
                 }
@@ -617,11 +733,11 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
                 Ok(true)
             }
             ty::FnPtr(..) => {
-                let value = self.read_scalar(value, ExpectedKind::FnPtr)?;
+                let scalar = self.read_scalar(value, ExpectedKind::FnPtr)?;
 
                 // If we check references recursively, also check that this points to a function.
                 if let Some(_) = self.ref_tracking {
-                    let ptr = value.to_pointer(self.ecx)?;
+                    let ptr = scalar.to_pointer(self.ecx)?;
                     let _fn = try_validation!(
                         self.ecx.get_ptr_fn(ptr),
                         self.path,
@@ -631,10 +747,18 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
                     // FIXME: Check if the signature matches
                 } else {
                     // Otherwise (for standalone Miri), we have to still check it to be non-null.
-                    if self.ecx.scalar_may_be_null(value)? {
+                    if self.ecx.scalar_may_be_null(scalar)? {
                         throw_validation_failure!(self.path, NullFnPtr);
                     }
                 }
+                if self.reset_provenance_and_padding {
+                    // Make sure we do not preserve partial provenance. This matches the thin
+                    // pointer handling in `deref_pointer`.
+                    if matches!(scalar, Scalar::Int(..)) {
+                        self.ecx.clear_provenance(value)?;
+                    }
+                    self.add_data_range_place(value);
+                }
                 Ok(true)
             }
             ty::Never => throw_validation_failure!(self.path, NeverVal),
@@ -716,13 +840,178 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
         }
     }
 
-    fn in_mutable_memory(&self, op: &OpTy<'tcx, M::Provenance>) -> bool {
-        if let Some(mplace) = op.as_mplace_or_imm().left() {
+    fn in_mutable_memory(&self, val: &PlaceTy<'tcx, M::Provenance>) -> bool {
+        if let Some(mplace) = val.as_mplace_or_local().left() {
             if let Some(alloc_id) = mplace.ptr().provenance.and_then(|p| p.get_alloc_id()) {
-                return mutability(self.ecx, alloc_id).is_mut();
+                mutability(self.ecx, alloc_id).is_mut()
+            } else {
+                // No memory at all.
+                false
+            }
+        } else {
+            // A local variable -- definitely mutable.
+            true
+        }
+    }
+
+    /// Add the given pointer-length pair to the "data" range of this visit.
+    fn add_data_range(&mut self, ptr: Pointer<Option<M::Provenance>>, size: Size) {
+        if let Some(data_bytes) = self.data_bytes.as_mut() {
+            // We only have to store the offset, the rest is the same for all pointers here.
+            let (_prov, offset) = ptr.into_parts();
+            // Add this.
+            data_bytes.add_range(offset, size);
+        };
+    }
+
+    /// Add the entire given place to the "data" range of this visit.
+    fn add_data_range_place(&mut self, place: &PlaceTy<'tcx, M::Provenance>) {
+        // Only sized places can be added this way.
+        debug_assert!(place.layout.abi.is_sized());
+        if let Some(data_bytes) = self.data_bytes.as_mut() {
+            let offset = Self::data_range_offset(self.ecx, place);
+            data_bytes.add_range(offset, place.layout.size);
+        }
+    }
+
+    /// Convert a place into the offset it starts at, for the purpose of data_range tracking.
+    /// Must only be called if `data_bytes` is `Some(_)`.
+    fn data_range_offset(ecx: &InterpCx<'tcx, M>, place: &PlaceTy<'tcx, M::Provenance>) -> Size {
+        // The presence of `data_bytes` implies that our place is in memory.
+        let ptr = ecx
+            .place_to_op(place)
+            .expect("place must be in memory")
+            .as_mplace_or_imm()
+            .expect_left("place must be in memory")
+            .ptr();
+        let (_prov, offset) = ptr.into_parts();
+        offset
+    }
+
+    fn reset_padding(&mut self, place: &PlaceTy<'tcx, M::Provenance>) -> InterpResult<'tcx> {
+        let Some(data_bytes) = self.data_bytes.as_mut() else { return Ok(()) };
+        // Our value must be in memory, otherwise we would not have set up `data_bytes`.
+        let mplace = self.ecx.force_allocation(place)?;
+        // Determine starting offset and size.
+        let (_prov, start_offset) = mplace.ptr().into_parts();
+        let (size, _align) = self
+            .ecx
+            .size_and_align_of_mplace(&mplace)?
+            .unwrap_or((mplace.layout.size, mplace.layout.align.abi));
+        // If there is no padding at all, we can skip the rest: check for
+        // a single data range covering the entire value.
+        if data_bytes.0 == &[(start_offset, size)] {
+            return Ok(());
+        }
+        // Get a handle for the allocation. Do this only once, to avoid looking up the same
+        // allocation over and over again. (Though to be fair, iterating the value already does
+        // exactly that.)
+        let Some(mut alloc) = self.ecx.get_ptr_alloc_mut(mplace.ptr(), size)? else {
+            // A ZST, no padding to clear.
+            return Ok(());
+        };
+        // Add a "finalizer" data range at the end, so that the iteration below finds all gaps
+        // between ranges.
+        data_bytes.0.push((start_offset + size, Size::ZERO));
+        // Iterate, and reset gaps.
+        let mut padding_cleared_until = start_offset;
+        for &(offset, size) in data_bytes.0.iter() {
+            assert!(
+                offset >= padding_cleared_until,
+                "reset_padding on {}: previous field ended at offset {}, next field starts at {} (and has a size of {} bytes)",
+                mplace.layout.ty,
+                (padding_cleared_until - start_offset).bytes(),
+                (offset - start_offset).bytes(),
+                size.bytes(),
+            );
+            if offset > padding_cleared_until {
+                // We found padding. Adjust the range to be relative to `alloc`, and make it uninit.
+                let padding_start = padding_cleared_until - start_offset;
+                let padding_size = offset - padding_cleared_until;
+                let range = alloc_range(padding_start, padding_size);
+                trace!("reset_padding on {}: resetting padding range {range:?}", mplace.layout.ty);
+                alloc.write_uninit(range)?;
+            }
+            padding_cleared_until = offset + size;
+        }
+        assert!(padding_cleared_until == start_offset + size);
+        Ok(())
+    }
+
+    /// Computes the data range of this union type:
+    /// which bytes are inside a field (i.e., not padding.)
+    fn union_data_range<'e>(
+        ecx: &'e mut InterpCx<'tcx, M>,
+        layout: TyAndLayout<'tcx>,
+    ) -> Cow<'e, RangeSet> {
+        assert!(layout.ty.is_union());
+        assert!(layout.abi.is_sized(), "there are no unsized unions");
+        let layout_cx = LayoutCx { tcx: *ecx.tcx, param_env: ecx.param_env };
+        return M::cached_union_data_range(ecx, layout.ty, || {
+            let mut out = RangeSet(Vec::new());
+            union_data_range_uncached(&layout_cx, layout, Size::ZERO, &mut out);
+            out
+        });
+
+        /// Helper for recursive traversal: add data ranges of the given type to `out`.
+        fn union_data_range_uncached<'tcx>(
+            cx: &LayoutCx<'tcx, TyCtxt<'tcx>>,
+            layout: TyAndLayout<'tcx>,
+            base_offset: Size,
+            out: &mut RangeSet,
+        ) {
+            // If this is a ZST, we don't contain any data. In particular, this helps us to quickly
+            // skip over huge arrays of ZST.
+            if layout.is_zst() {
+                return;
+            }
+            // Just recursively add all the fields of everything to the output.
+            match &layout.fields {
+                FieldsShape::Primitive => {
+                    out.add_range(base_offset, layout.size);
+                }
+                &FieldsShape::Union(fields) => {
+                    // Currently, all fields start at offset 0 (relative to `base_offset`).
+                    for field in 0..fields.get() {
+                        let field = layout.field(cx, field);
+                        union_data_range_uncached(cx, field, base_offset, out);
+                    }
+                }
+                &FieldsShape::Array { stride, count } => {
+                    let elem = layout.field(cx, 0);
+
+                    // Fast-path for large arrays of simple types that do not contain any padding.
+                    if elem.abi.is_scalar() {
+                        out.add_range(base_offset, elem.size * count);
+                    } else {
+                        for idx in 0..count {
+                            // This repeats the same computation for every array element... but the alternative
+                            // is to allocate temporary storage for a dedicated `out` set for the array element,
+                            // and replicating that N times. Is that better?
+                            union_data_range_uncached(cx, elem, base_offset + idx * stride, out);
+                        }
+                    }
+                }
+                FieldsShape::Arbitrary { offsets, .. } => {
+                    for (field, &offset) in offsets.iter_enumerated() {
+                        let field = layout.field(cx, field.as_usize());
+                        union_data_range_uncached(cx, field, base_offset + offset, out);
+                    }
+                }
+            }
+            // Don't forget potential other variants.
+            match &layout.variants {
+                Variants::Single { .. } => {
+                    // Fully handled above.
+                }
+                Variants::Multiple { variants, .. } => {
+                    for variant in variants.indices() {
+                        let variant = layout.for_variant(cx, variant);
+                        union_data_range_uncached(cx, variant, base_offset, out);
+                    }
+                }
             }
         }
-        false
     }
 }
 
@@ -774,7 +1063,7 @@ fn mutability<'tcx>(ecx: &InterpCx<'tcx, impl Machine<'tcx>>, alloc_id: AllocId)
 }
 
 impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt, 'tcx, M> {
-    type V = OpTy<'tcx, M::Provenance>;
+    type V = PlaceTy<'tcx, M::Provenance>;
 
     #[inline(always)]
     fn ecx(&self) -> &InterpCx<'tcx, M> {
@@ -783,11 +1072,11 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt,
 
     fn read_discriminant(
         &mut self,
-        op: &OpTy<'tcx, M::Provenance>,
+        val: &PlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx, VariantIdx> {
         self.with_elem(PathElem::EnumTag, move |this| {
             Ok(try_validation!(
-                this.ecx.read_discriminant(op),
+                this.ecx.read_discriminant(val),
                 this.path,
                 Ub(InvalidTag(val)) => InvalidEnumTag {
                     value: format!("{val:x}"),
@@ -802,44 +1091,54 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt,
     #[inline]
     fn visit_field(
         &mut self,
-        old_op: &OpTy<'tcx, M::Provenance>,
+        old_val: &PlaceTy<'tcx, M::Provenance>,
         field: usize,
-        new_op: &OpTy<'tcx, M::Provenance>,
+        new_val: &PlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx> {
-        let elem = self.aggregate_field_path_elem(old_op.layout, field);
-        self.with_elem(elem, move |this| this.visit_value(new_op))
+        let elem = self.aggregate_field_path_elem(old_val.layout, field);
+        self.with_elem(elem, move |this| this.visit_value(new_val))
     }
 
     #[inline]
     fn visit_variant(
         &mut self,
-        old_op: &OpTy<'tcx, M::Provenance>,
+        old_val: &PlaceTy<'tcx, M::Provenance>,
         variant_id: VariantIdx,
-        new_op: &OpTy<'tcx, M::Provenance>,
+        new_val: &PlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx> {
-        let name = match old_op.layout.ty.kind() {
+        let name = match old_val.layout.ty.kind() {
             ty::Adt(adt, _) => PathElem::Variant(adt.variant(variant_id).name),
             // Coroutines also have variants
             ty::Coroutine(..) => PathElem::CoroutineState(variant_id),
-            _ => bug!("Unexpected type with variant: {:?}", old_op.layout.ty),
+            _ => bug!("Unexpected type with variant: {:?}", old_val.layout.ty),
         };
-        self.with_elem(name, move |this| this.visit_value(new_op))
+        self.with_elem(name, move |this| this.visit_value(new_val))
     }
 
     #[inline(always)]
     fn visit_union(
         &mut self,
-        op: &OpTy<'tcx, M::Provenance>,
+        val: &PlaceTy<'tcx, M::Provenance>,
         _fields: NonZero<usize>,
     ) -> InterpResult<'tcx> {
         // Special check for CTFE validation, preventing `UnsafeCell` inside unions in immutable memory.
         if self.ctfe_mode.is_some_and(|c| !c.allow_immutable_unsafe_cell()) {
-            if !op.layout.is_zst() && !op.layout.ty.is_freeze(*self.ecx.tcx, self.ecx.param_env) {
-                if !self.in_mutable_memory(op) {
+            if !val.layout.is_zst() && !val.layout.ty.is_freeze(*self.ecx.tcx, self.ecx.param_env) {
+                if !self.in_mutable_memory(val) {
                     throw_validation_failure!(self.path, UnsafeCellInImmutable);
                 }
             }
         }
+        if self.reset_provenance_and_padding
+            && let Some(data_bytes) = self.data_bytes.as_mut()
+        {
+            let base_offset = Self::data_range_offset(self.ecx, val);
+            // Determine and add data range for this union.
+            let union_data_range = Self::union_data_range(self.ecx, val.layout);
+            for &(offset, size) in union_data_range.0.iter() {
+                data_bytes.add_range(base_offset + offset, size);
+            }
+        }
         Ok(())
     }
 
@@ -847,39 +1146,41 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt,
     fn visit_box(
         &mut self,
         _box_ty: Ty<'tcx>,
-        op: &OpTy<'tcx, M::Provenance>,
+        val: &PlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx> {
-        self.check_safe_pointer(op, PointerKind::Box)?;
+        self.check_safe_pointer(val, PointerKind::Box)?;
         Ok(())
     }
 
     #[inline]
-    fn visit_value(&mut self, op: &OpTy<'tcx, M::Provenance>) -> InterpResult<'tcx> {
-        trace!("visit_value: {:?}, {:?}", *op, op.layout);
+    fn visit_value(&mut self, val: &PlaceTy<'tcx, M::Provenance>) -> InterpResult<'tcx> {
+        trace!("visit_value: {:?}, {:?}", *val, val.layout);
 
         // Check primitive types -- the leaves of our recursive descent.
+        // This is called even for enum discriminants (which are "fields" of their enum),
+        // so for integer-typed discriminants the provenance reset will happen here.
         // We assume that the Scalar validity range does not restrict these values
         // any further than `try_visit_primitive` does!
-        if self.try_visit_primitive(op)? {
+        if self.try_visit_primitive(val)? {
             return Ok(());
         }
 
         // Special check preventing `UnsafeCell` in the inner part of constants
         if self.ctfe_mode.is_some_and(|c| !c.allow_immutable_unsafe_cell()) {
-            if !op.layout.is_zst()
-                && let Some(def) = op.layout.ty.ty_adt_def()
+            if !val.layout.is_zst()
+                && let Some(def) = val.layout.ty.ty_adt_def()
                 && def.is_unsafe_cell()
             {
-                if !self.in_mutable_memory(op) {
+                if !self.in_mutable_memory(val) {
                     throw_validation_failure!(self.path, UnsafeCellInImmutable);
                 }
             }
         }
 
         // Recursively walk the value at its type. Apply optimizations for some large types.
-        match op.layout.ty.kind() {
+        match val.layout.ty.kind() {
             ty::Str => {
-                let mplace = op.assert_mem_place(); // strings are unsized and hence never immediate
+                let mplace = val.assert_mem_place(); // strings are unsized and hence never immediate
                 let len = mplace.len(self.ecx)?;
                 try_validation!(
                     self.ecx.read_bytes_ptr_strip_provenance(mplace.ptr(), Size::from_bytes(len)),
@@ -889,11 +1190,10 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt,
                 );
             }
             ty::Array(tys, ..) | ty::Slice(tys)
-                // This optimization applies for types that can hold arbitrary bytes (such as
-                // integer and floating point types) or for structs or tuples with no fields.
-                // FIXME(wesleywiser) This logic could be extended further to arbitrary structs
-                // or tuples made up of integer/floating point types or inhabited ZSTs with no
-                // padding.
+                // This optimization applies for types that can hold arbitrary non-provenance bytes (such as
+                // integer and floating point types).
+                // FIXME(wesleywiser) This logic could be extended further to arbitrary structs or
+                // tuples made up of integer/floating point types or inhabited ZSTs with no padding.
                 if matches!(tys.kind(), ty::Int(..) | ty::Uint(..) | ty::Float(..))
                 =>
             {
@@ -901,18 +1201,19 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt,
                 // Optimized handling for arrays of integer/float type.
 
                 // This is the length of the array/slice.
-                let len = op.len(self.ecx)?;
+                let len = val.len(self.ecx)?;
                 // This is the element type size.
                 let layout = self.ecx.layout_of(*tys)?;
                 // This is the size in bytes of the whole array. (This checks for overflow.)
                 let size = layout.size * len;
                 // If the size is 0, there is nothing to check.
-                // (`size` can only be 0 of `len` is 0, and empty arrays are always valid.)
+                // (`size` can only be 0 if `len` is 0, and empty arrays are always valid.)
                 if size == Size::ZERO {
                     return Ok(());
                 }
-                // Now that we definitely have a non-ZST array, we know it lives in memory.
-                let mplace = match op.as_mplace_or_imm() {
+                // Now that we definitely have a non-ZST array, we know it lives in memory -- except it may
+                // be an uninitialized local variable, those are also "immediate".
+                let mplace = match val.to_op(self.ecx)?.as_mplace_or_imm() {
                     Left(mplace) => mplace,
                     Right(imm) => match *imm {
                         Immediate::Uninit =>
@@ -958,20 +1259,30 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt,
                         }
                     }
                 }
+
+                // Don't forget that these are all non-pointer types, and thus do not preserve
+                // provenance.
+                if self.reset_provenance_and_padding {
+                    // We can't share this with above as above, we might be looking at read-only memory.
+                    let mut alloc = self.ecx.get_ptr_alloc_mut(mplace.ptr(), size)?.expect("we already excluded size 0");
+                    alloc.clear_provenance()?;
+                    // Also, mark this as containing data, not padding.
+                    self.add_data_range(mplace.ptr(), size);
+                }
             }
             // Fast path for arrays and slices of ZSTs. We only need to check a single ZST element
             // of an array and not all of them, because there's only a single value of a specific
             // ZST type, so either validation fails for all elements or none.
             ty::Array(tys, ..) | ty::Slice(tys) if self.ecx.layout_of(*tys)?.is_zst() => {
                 // Validate just the first element (if any).
-                if op.len(self.ecx)? > 0 {
-                    self.visit_field(op, 0, &self.ecx.project_index(op, 0)?)?;
+                if val.len(self.ecx)? > 0 {
+                    self.visit_field(val, 0, &self.ecx.project_index(val, 0)?)?;
                 }
             }
             _ => {
                 // default handler
                 try_validation!(
-                    self.walk_value(op),
+                    self.walk_value(val),
                     self.path,
                     // It's not great to catch errors here, since we can't give a very good path,
                     // but it's better than ICEing.
@@ -992,15 +1303,15 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt,
         // FIXME: We could avoid some redundant checks here. For newtypes wrapping
         // scalars, we do the same check on every "level" (e.g., first we check
         // MyNewtype and then the scalar in there).
-        match op.layout.abi {
+        match val.layout.abi {
             Abi::Uninhabited => {
-                let ty = op.layout.ty;
+                let ty = val.layout.ty;
                 throw_validation_failure!(self.path, UninhabitedVal { ty });
             }
             Abi::Scalar(scalar_layout) => {
                 if !scalar_layout.is_uninit_valid() {
                     // There is something to check here.
-                    let scalar = self.read_scalar(op, ExpectedKind::InitScalar)?;
+                    let scalar = self.read_scalar(val, ExpectedKind::InitScalar)?;
                     self.visit_scalar(scalar, scalar_layout)?;
                 }
             }
@@ -1010,7 +1321,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt,
                 // the other must be init.
                 if !a_layout.is_uninit_valid() && !b_layout.is_uninit_valid() {
                     let (a, b) =
-                        self.read_immediate(op, ExpectedKind::InitScalar)?.to_scalar_pair();
+                        self.read_immediate(val, ExpectedKind::InitScalar)?.to_scalar_pair();
                     self.visit_scalar(a, a_layout)?;
                     self.visit_scalar(b, b_layout)?;
                 }
@@ -1031,19 +1342,34 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt,
 
 impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     fn validate_operand_internal(
-        &self,
-        op: &OpTy<'tcx, M::Provenance>,
+        &mut self,
+        val: &PlaceTy<'tcx, M::Provenance>,
         path: Vec<PathElem>,
         ref_tracking: Option<&mut RefTracking<MPlaceTy<'tcx, M::Provenance>, Vec<PathElem>>>,
         ctfe_mode: Option<CtfeValidationMode>,
+        reset_provenance_and_padding: bool,
     ) -> InterpResult<'tcx> {
-        trace!("validate_operand_internal: {:?}, {:?}", *op, op.layout.ty);
+        trace!("validate_operand_internal: {:?}, {:?}", *val, val.layout.ty);
 
-        // Construct a visitor
-        let mut visitor = ValidityVisitor { path, ref_tracking, ctfe_mode, ecx: self };
-
-        // Run it.
-        match self.run_for_validation(|| visitor.visit_value(op)) {
+        // Run the visitor.
+        match self.run_for_validation(|ecx| {
+            let reset_padding = reset_provenance_and_padding && {
+                // Check if `val` is actually stored in memory. If not, padding is not even
+                // represented and we need not reset it.
+                ecx.place_to_op(val)?.as_mplace_or_imm().is_left()
+            };
+            let mut v = ValidityVisitor {
+                path,
+                ref_tracking,
+                ctfe_mode,
+                ecx,
+                reset_provenance_and_padding,
+                data_bytes: reset_padding.then_some(RangeSet(Vec::new())),
+            };
+            v.visit_value(val)?;
+            v.reset_padding(val)?;
+            InterpResult::Ok(())
+        }) {
             Ok(()) => Ok(()),
             // Pass through validation failures and "invalid program" issues.
             Err(err)
@@ -1079,13 +1405,19 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     /// - no `UnsafeCell` or non-ZST `&mut`.
     #[inline(always)]
     pub(crate) fn const_validate_operand(
-        &self,
-        op: &OpTy<'tcx, M::Provenance>,
+        &mut self,
+        val: &PlaceTy<'tcx, M::Provenance>,
         path: Vec<PathElem>,
         ref_tracking: &mut RefTracking<MPlaceTy<'tcx, M::Provenance>, Vec<PathElem>>,
         ctfe_mode: CtfeValidationMode,
     ) -> InterpResult<'tcx> {
-        self.validate_operand_internal(op, path, Some(ref_tracking), Some(ctfe_mode))
+        self.validate_operand_internal(
+            val,
+            path,
+            Some(ref_tracking),
+            Some(ctfe_mode),
+            /*reset_provenance*/ false,
+        )
     }
 
     /// This function checks the data at `op` to be runtime-valid.
@@ -1093,21 +1425,41 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     /// It will error if the bits at the destination do not match the ones described by the layout.
     #[inline(always)]
     pub fn validate_operand(
-        &self,
-        op: &OpTy<'tcx, M::Provenance>,
+        &mut self,
+        val: &PlaceTy<'tcx, M::Provenance>,
         recursive: bool,
+        reset_provenance_and_padding: bool,
     ) -> InterpResult<'tcx> {
         // Note that we *could* actually be in CTFE here with `-Zextra-const-ub-checks`, but it's
         // still correct to not use `ctfe_mode`: that mode is for validation of the final constant
         // value, it rules out things like `UnsafeCell` in awkward places.
         if !recursive {
-            return self.validate_operand_internal(op, vec![], None, None);
+            return self.validate_operand_internal(
+                val,
+                vec![],
+                None,
+                None,
+                reset_provenance_and_padding,
+            );
         }
         // Do a recursive check.
         let mut ref_tracking = RefTracking::empty();
-        self.validate_operand_internal(op, vec![], Some(&mut ref_tracking), None)?;
+        self.validate_operand_internal(
+            val,
+            vec![],
+            Some(&mut ref_tracking),
+            None,
+            reset_provenance_and_padding,
+        )?;
         while let Some((mplace, path)) = ref_tracking.todo.pop() {
-            self.validate_operand_internal(&mplace.into(), path, Some(&mut ref_tracking), None)?;
+            // Things behind reference do *not* have the provenance reset.
+            self.validate_operand_internal(
+                &mplace.into(),
+                path,
+                Some(&mut ref_tracking),
+                None,
+                /*reset_provenance_and_padding*/ false,
+            )?;
         }
         Ok(())
     }
diff --git a/compiler/rustc_const_eval/src/interpret/visitor.rs b/compiler/rustc_const_eval/src/interpret/visitor.rs
index b02f12e3c7f..d8af67bd0e7 100644
--- a/compiler/rustc_const_eval/src/interpret/visitor.rs
+++ b/compiler/rustc_const_eval/src/interpret/visitor.rs
@@ -5,6 +5,7 @@ use std::num::NonZero;
 
 use rustc_index::IndexVec;
 use rustc_middle::mir::interpret::InterpResult;
+use rustc_middle::ty::layout::LayoutOf;
 use rustc_middle::ty::{self, Ty};
 use rustc_target::abi::{FieldIdx, FieldsShape, VariantIdx, Variants};
 use tracing::trace;
@@ -82,6 +83,7 @@ pub trait ValueVisitor<'tcx, M: Machine<'tcx>>: Sized {
         self.visit_value(new_val)
     }
 
+    /// Traversal logic; should not be overloaded.
     fn walk_value(&mut self, v: &Self::V) -> InterpResult<'tcx> {
         let ty = v.layout().ty;
         trace!("walk_value: type: {ty}");
@@ -104,6 +106,17 @@ pub trait ValueVisitor<'tcx, M: Machine<'tcx>>: Sized {
                 // DynStar types. Very different from a dyn type (but strangely part of the
                 // same variant in `TyKind`): These are pairs where the 2nd component is the
                 // vtable, and the first component is the data (which must be ptr-sized).
+
+                // First make sure the vtable can be read at its type.
+                // The type of this vtable is fake, it claims to be a reference to some actual memory but that isn't true.
+                // So we transmute it to a raw pointer.
+                let raw_ptr_ty = Ty::new_mut_ptr(*self.ecx().tcx, self.ecx().tcx.types.unit);
+                let raw_ptr_ty = self.ecx().layout_of(raw_ptr_ty)?;
+                let vtable_field =
+                    self.ecx().project_field(v, 1)?.transmute(raw_ptr_ty, self.ecx())?;
+                self.visit_field(v, 1, &vtable_field)?;
+
+                // Then unpack the first field, and continue.
                 let data = self.ecx().unpack_dyn_star(v, data)?;
                 return self.visit_field(v, 0, &data);
             }
diff --git a/compiler/rustc_const_eval/src/util/check_validity_requirement.rs b/compiler/rustc_const_eval/src/util/check_validity_requirement.rs
index cbd1fdeea2a..611a8e1a884 100644
--- a/compiler/rustc_const_eval/src/util/check_validity_requirement.rs
+++ b/compiler/rustc_const_eval/src/util/check_validity_requirement.rs
@@ -4,7 +4,7 @@ use rustc_middle::ty::{ParamEnvAnd, Ty, TyCtxt};
 use rustc_target::abi::{Abi, FieldsShape, Scalar, Variants};
 
 use crate::const_eval::{CanAccessMutGlobal, CheckAlignment, CompileTimeMachine};
-use crate::interpret::{InterpCx, MemoryKind, OpTy};
+use crate::interpret::{InterpCx, MemoryKind};
 
 /// Determines if this type permits "raw" initialization by just transmuting some memory into an
 /// instance of `T`.
@@ -32,15 +32,15 @@ pub fn check_validity_requirement<'tcx>(
 
     let layout_cx = LayoutCx { tcx, param_env: param_env_and_ty.param_env };
     if kind == ValidityRequirement::Uninit || tcx.sess.opts.unstable_opts.strict_init_checks {
-        might_permit_raw_init_strict(layout, &layout_cx, kind)
+        check_validity_requirement_strict(layout, &layout_cx, kind)
     } else {
-        might_permit_raw_init_lax(layout, &layout_cx, kind)
+        check_validity_requirement_lax(layout, &layout_cx, kind)
     }
 }
 
-/// Implements the 'strict' version of the `might_permit_raw_init` checks; see that function for
-/// details.
-fn might_permit_raw_init_strict<'tcx>(
+/// Implements the 'strict' version of the [`check_validity_requirement`] checks; see that function
+/// for details.
+fn check_validity_requirement_strict<'tcx>(
     ty: TyAndLayout<'tcx>,
     cx: &LayoutCx<'tcx, TyCtxt<'tcx>>,
     kind: ValidityRequirement,
@@ -61,18 +61,24 @@ fn might_permit_raw_init_strict<'tcx>(
         .expect("failed to write bytes for zero valid check");
     }
 
-    let ot: OpTy<'_, _> = allocated.into();
-
     // Assume that if it failed, it's a validation failure.
     // This does *not* actually check that references are dereferenceable, but since all types that
     // require dereferenceability also require non-null, we don't actually get any false negatives
     // due to this.
-    Ok(cx.validate_operand(&ot, /*recursive*/ false).is_ok())
+    // The value we are validating is temporary and discarded at the end of this function, so
+    // there is no point in reseting provenance and padding.
+    Ok(cx
+        .validate_operand(
+            &allocated.into(),
+            /*recursive*/ false,
+            /*reset_provenance_and_padding*/ false,
+        )
+        .is_ok())
 }
 
-/// Implements the 'lax' (default) version of the `might_permit_raw_init` checks; see that function for
-/// details.
-fn might_permit_raw_init_lax<'tcx>(
+/// Implements the 'lax' (default) version of the [`check_validity_requirement`] checks; see that
+/// function for details.
+fn check_validity_requirement_lax<'tcx>(
     this: TyAndLayout<'tcx>,
     cx: &LayoutCx<'tcx, TyCtxt<'tcx>>,
     init_kind: ValidityRequirement,
@@ -137,7 +143,7 @@ fn might_permit_raw_init_lax<'tcx>(
         }
         FieldsShape::Arbitrary { offsets, .. } => {
             for idx in 0..offsets.len() {
-                if !might_permit_raw_init_lax(this.field(cx, idx), cx, init_kind)? {
+                if !check_validity_requirement_lax(this.field(cx, idx), cx, init_kind)? {
                     // We found a field that is unhappy with this kind of initialization.
                     return Ok(false);
                 }
diff --git a/compiler/rustc_data_structures/src/steal.rs b/compiler/rustc_data_structures/src/steal.rs
index 0f2c0eee27d..aaa95f6b7f1 100644
--- a/compiler/rustc_data_structures/src/steal.rs
+++ b/compiler/rustc_data_structures/src/steal.rs
@@ -57,6 +57,7 @@ impl<T> Steal<T> {
     ///
     /// This should not be used within rustc as it leaks information not tracked
     /// by the query system, breaking incremental compilation.
+    #[cfg_attr(not(bootstrap), rustc_lint_untracked_query_information)]
     pub fn is_stolen(&self) -> bool {
         self.value.borrow().is_none()
     }
diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs
index cb2fa6e9d74..1b7ca61cee8 100644
--- a/compiler/rustc_driver_impl/src/lib.rs
+++ b/compiler/rustc_driver_impl/src/lib.rs
@@ -7,7 +7,6 @@
 // tidy-alphabetical-start
 #![allow(internal_features)]
 #![allow(rustc::untranslatable_diagnostic)] // FIXME: make this translatable
-#![cfg_attr(bootstrap, feature(unsafe_extern_blocks))]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(decl_macro)]
@@ -862,9 +861,9 @@ fn print_crate_info(
                 use rustc_target::spec::current_apple_deployment_target;
 
                 if sess.target.is_like_osx {
-                    let (major, minor) = current_apple_deployment_target(&sess.target)
-                        .expect("unknown Apple target OS");
-                    println_info!("deployment_target={}", format!("{major}.{minor}"))
+                    let (major, minor, patch) = current_apple_deployment_target(&sess.target);
+                    let patch = if patch != 0 { format!(".{patch}") } else { String::new() };
+                    println_info!("deployment_target={major}.{minor}{patch}")
                 } else {
                     #[allow(rustc::diagnostic_outside_of_impl)]
                     sess.dcx().fatal("only Apple targets currently support deployment version info")
diff --git a/compiler/rustc_expand/src/mbe/macro_parser.rs b/compiler/rustc_expand/src/mbe/macro_parser.rs
index 9011d02da33..501a2417fcf 100644
--- a/compiler/rustc_expand/src/mbe/macro_parser.rs
+++ b/compiler/rustc_expand/src/mbe/macro_parser.rs
@@ -398,8 +398,10 @@ pub(crate) enum NamedMatch {
 fn token_name_eq(t1: &Token, t2: &Token) -> bool {
     if let (Some((ident1, is_raw1)), Some((ident2, is_raw2))) = (t1.ident(), t2.ident()) {
         ident1.name == ident2.name && is_raw1 == is_raw2
-    } else if let (Some(ident1), Some(ident2)) = (t1.lifetime(), t2.lifetime()) {
-        ident1.name == ident2.name
+    } else if let (Some((ident1, is_raw1)), Some((ident2, is_raw2))) =
+        (t1.lifetime(), t2.lifetime())
+    {
+        ident1.name == ident2.name && is_raw1 == is_raw2
     } else {
         t1.kind == t2.kind
     }
diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs
index b06910595bb..39489a8df1b 100644
--- a/compiler/rustc_expand/src/mbe/transcribe.rs
+++ b/compiler/rustc_expand/src/mbe/transcribe.rs
@@ -283,9 +283,9 @@ pub(super) fn transcribe<'a>(
                             let kind = token::NtIdent(*ident, *is_raw);
                             TokenTree::token_alone(kind, sp)
                         }
-                        MatchedSingle(ParseNtResult::Lifetime(ident)) => {
+                        MatchedSingle(ParseNtResult::Lifetime(ident, is_raw)) => {
                             marker.visit_span(&mut sp);
-                            let kind = token::NtLifetime(*ident);
+                            let kind = token::NtLifetime(*ident, *is_raw);
                             TokenTree::token_alone(kind, sp)
                         }
                         MatchedSingle(ParseNtResult::Nt(nt)) => {
diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs
index 4ff5da1a4bd..5798bcedc22 100644
--- a/compiler/rustc_expand/src/proc_macro_server.rs
+++ b/compiler/rustc_expand/src/proc_macro_server.rs
@@ -229,15 +229,16 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
                     span: ident.span,
                 })),
 
-                Lifetime(name) => {
+                Lifetime(name, is_raw) => {
                     let ident = symbol::Ident::new(name, span).without_first_quote();
                     trees.extend([
                         TokenTree::Punct(Punct { ch: b'\'', joint: true, span }),
-                        TokenTree::Ident(Ident { sym: ident.name, is_raw: false, span }),
+                        TokenTree::Ident(Ident { sym: ident.name, is_raw: is_raw.into(), span }),
                     ]);
                 }
-                NtLifetime(ident) => {
-                    let stream = TokenStream::token_alone(token::Lifetime(ident.name), ident.span);
+                NtLifetime(ident, is_raw) => {
+                    let stream =
+                        TokenStream::token_alone(token::Lifetime(ident.name, is_raw), ident.span);
                     trees.push(TokenTree::Group(Group {
                         delimiter: pm::Delimiter::None,
                         stream: Some(stream),
diff --git a/compiler/rustc_feature/src/accepted.rs b/compiler/rustc_feature/src/accepted.rs
index 9223c3c322a..8949fdffdae 100644
--- a/compiler/rustc_feature/src/accepted.rs
+++ b/compiler/rustc_feature/src/accepted.rs
@@ -61,7 +61,7 @@ declare_features! (
     /// Allows explicit discriminants on non-unit enum variants.
     (accepted, arbitrary_enum_discriminant, "1.66.0", Some(60553)),
     /// Allows using `const` operands in inline assembly.
-    (accepted, asm_const, "CURRENT_RUSTC_VERSION", Some(93332)),
+    (accepted, asm_const, "1.82.0", Some(93332)),
     /// Allows using `sym` operands in inline assembly.
     (accepted, asm_sym, "1.66.0", Some(93333)),
     /// Allows the definition of associated constants in `trait` or `impl` blocks.
@@ -116,7 +116,7 @@ declare_features! (
     /// Allows calling constructor functions in `const fn`.
     (accepted, const_constructor, "1.40.0", Some(61456)),
     /// Allows basic arithmetic on floating point types in a `const fn`.
-    (accepted, const_fn_floating_point_arithmetic, "CURRENT_RUSTC_VERSION", Some(57241)),
+    (accepted, const_fn_floating_point_arithmetic, "1.82.0", Some(57241)),
     /// Allows using and casting function pointers in a `const fn`.
     (accepted, const_fn_fn_ptr_basics, "1.61.0", Some(57563)),
     /// Allows trait bounds in `const fn`.
@@ -272,7 +272,7 @@ declare_features! (
     /// Allows calling `const unsafe fn` inside `unsafe` blocks in `const fn` functions.
     (accepted, min_const_unsafe_fn, "1.33.0", Some(55607)),
     /// Allows exhaustive pattern matching on uninhabited types when matched by value.
-    (accepted, min_exhaustive_patterns, "CURRENT_RUSTC_VERSION", Some(119612)),
+    (accepted, min_exhaustive_patterns, "1.82.0", Some(119612)),
     /// Allows using `Self` and associated types in struct expressions and patterns.
     (accepted, more_struct_aliases, "1.16.0", Some(37544)),
     /// Allows using the MOVBE target feature.
@@ -299,7 +299,7 @@ declare_features! (
     /// Allows `foo.rs` as an alternative to `foo/mod.rs`.
     (accepted, non_modrs_mods, "1.30.0", Some(44660)),
     /// Allows using multiple nested field accesses in offset_of!
-    (accepted, offset_of_nested, "CURRENT_RUSTC_VERSION", Some(120140)),
+    (accepted, offset_of_nested, "1.82.0", Some(120140)),
     /// Allows the use of or-patterns (e.g., `0 | 1`).
     (accepted, or_patterns, "1.53.0", Some(54883)),
     /// Allows using `+bundle,+whole-archive` link modifiers with native libs.
@@ -312,7 +312,7 @@ declare_features! (
     /// Allows parentheses in patterns.
     (accepted, pattern_parentheses, "1.31.0", Some(51087)),
     /// Allows `use<'a, 'b, A, B>` in `impl Trait + use<...>` for precise capture of generic args.
-    (accepted, precise_capturing, "CURRENT_RUSTC_VERSION", Some(123432)),
+    (accepted, precise_capturing, "1.82.0", Some(123432)),
     /// Allows procedural macros in `proc-macro` crates.
     (accepted, proc_macro, "1.29.0", Some(38356)),
     /// Allows multi-segment paths in attributes and derives.
@@ -326,7 +326,7 @@ declare_features! (
     /// Allows keywords to be escaped for use as identifiers.
     (accepted, raw_identifiers, "1.30.0", Some(48589)),
     /// Allows `&raw const $place_expr` and `&raw mut $place_expr` expressions.
-    (accepted, raw_ref_op, "CURRENT_RUSTC_VERSION", Some(64490)),
+    (accepted, raw_ref_op, "1.82.0", Some(64490)),
     /// Allows relaxing the coherence rules such that
     /// `impl<T> ForeignTrait<LocalType> for ForeignType<T>` is permitted.
     (accepted, re_rebalance_coherence, "1.41.0", Some(55437)),
@@ -399,11 +399,11 @@ declare_features! (
     /// Allows arbitrary delimited token streams in non-macro attributes.
     (accepted, unrestricted_attribute_tokens, "1.34.0", Some(55208)),
     /// Allows unsafe attributes.
-    (accepted, unsafe_attributes, "CURRENT_RUSTC_VERSION", Some(123757)),
+    (accepted, unsafe_attributes, "1.82.0", Some(123757)),
     /// The `unsafe_op_in_unsafe_fn` lint (allowed by default): no longer treat an unsafe function as an unsafe block.
     (accepted, unsafe_block_in_unsafe_fn, "1.52.0", Some(71668)),
     /// Allows unsafe on extern declarations and safety qualifiers over internal items.
-    (accepted, unsafe_extern_blocks, "CURRENT_RUSTC_VERSION", Some(123743)),
+    (accepted, unsafe_extern_blocks, "1.82.0", Some(123743)),
     /// Allows importing and reexporting macros with `use`,
     /// enables macro modularization in general.
     (accepted, use_extern_macros, "1.30.0", Some(35896)),
diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs
index e2491922b8d..3b7e0d82d0f 100644
--- a/compiler/rustc_feature/src/builtin_attrs.rs
+++ b/compiler/rustc_feature/src/builtin_attrs.rs
@@ -793,6 +793,12 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
         rustc_lint_query_instability, Normal, template!(Word),
         WarnFollowing, EncodeCrossCrate::Yes, INTERNAL_UNSTABLE
     ),
+    // Used by the `rustc::untracked_query_information` lint to warn methods which
+    // might not be stable during incremental compilation.
+    rustc_attr!(
+        rustc_lint_untracked_query_information, Normal, template!(Word),
+        WarnFollowing, EncodeCrossCrate::Yes, INTERNAL_UNSTABLE
+    ),
     // Used by the `rustc::diagnostic_outside_of_impl` lints to assist in changes to diagnostic
     // APIs. Any function with this attribute will be checked by that lint.
     rustc_attr!(
@@ -1180,3 +1186,11 @@ pub static BUILTIN_ATTRIBUTE_MAP: LazyLock<FxHashMap<Symbol, &BuiltinAttribute>>
         }
         map
     });
+
+pub fn is_stable_diagnostic_attribute(sym: Symbol, features: &Features) -> bool {
+    match sym {
+        sym::on_unimplemented => true,
+        sym::do_not_recommend => features.do_not_recommend,
+        _ => false,
+    }
+}
diff --git a/compiler/rustc_feature/src/lib.rs b/compiler/rustc_feature/src/lib.rs
index adaaba3cd23..fe12930e6b9 100644
--- a/compiler/rustc_feature/src/lib.rs
+++ b/compiler/rustc_feature/src/lib.rs
@@ -130,8 +130,9 @@ pub fn find_feature_issue(feature: Symbol, issue: GateIssue) -> Option<NonZero<u
 pub use accepted::ACCEPTED_FEATURES;
 pub use builtin_attrs::{
     deprecated_attributes, encode_cross_crate, find_gated_cfg, is_builtin_attr_name,
-    is_valid_for_get_attr, AttributeDuplicates, AttributeGate, AttributeSafety, AttributeTemplate,
-    AttributeType, BuiltinAttribute, GatedCfg, BUILTIN_ATTRIBUTES, BUILTIN_ATTRIBUTE_MAP,
+    is_stable_diagnostic_attribute, is_valid_for_get_attr, AttributeDuplicates, AttributeGate,
+    AttributeSafety, AttributeTemplate, AttributeType, BuiltinAttribute, GatedCfg,
+    BUILTIN_ATTRIBUTES, BUILTIN_ATTRIBUTE_MAP,
 };
 pub use removed::REMOVED_FEATURES;
 pub use unstable::{Features, INCOMPATIBLE_FEATURES, UNSTABLE_FEATURES};
diff --git a/compiler/rustc_feature/src/removed.rs b/compiler/rustc_feature/src/removed.rs
index b7f0ed5afce..3d7d877ba17 100644
--- a/compiler/rustc_feature/src/removed.rs
+++ b/compiler/rustc_feature/src/removed.rs
@@ -83,7 +83,7 @@ declare_features! (
     (removed, custom_derive, "1.32.0", Some(29644),
      Some("subsumed by `#[proc_macro_derive]`")),
     /// Allows default type parameters to influence type inference.
-    (removed, default_type_parameter_fallback, "CURRENT_RUSTC_VERSION", Some(27336),
+    (removed, default_type_parameter_fallback, "1.82.0", Some(27336),
      Some("never properly implemented; requires significant design work")),
     /// Allows using `#[doc(keyword = "...")]`.
     (removed, doc_keyword, "1.28.0", Some(51315),
diff --git a/compiler/rustc_feature/src/unstable.rs b/compiler/rustc_feature/src/unstable.rs
index fba65883550..3254dab9a03 100644
--- a/compiler/rustc_feature/src/unstable.rs
+++ b/compiler/rustc_feature/src/unstable.rs
@@ -208,7 +208,7 @@ declare_features! (
     /// Allows `#[link(..., cfg(..))]`; perma-unstable per #37406
     (unstable, link_cfg, "1.14.0", None),
     /// Allows using `?Trait` trait bounds in more contexts.
-    (internal, more_maybe_bounds, "CURRENT_RUSTC_VERSION", None),
+    (internal, more_maybe_bounds, "1.82.0", None),
     /// Allows the `multiple_supertrait_upcastable` lint.
     (unstable, multiple_supertrait_upcastable, "1.69.0", None),
     /// Allow negative trait bounds. This is an internal-only feature for testing the trait solver!
@@ -302,7 +302,7 @@ declare_features! (
     // FIXME: Document these and merge with the list below.
 
     // Unstable `#[target_feature]` directives.
-    (unstable, aarch64_unstable_target_feature, "CURRENT_RUSTC_VERSION", Some(44839)),
+    (unstable, aarch64_unstable_target_feature, "1.82.0", Some(44839)),
     (unstable, aarch64_ver_target_feature, "1.27.0", Some(44839)),
     (unstable, arm_target_feature, "1.27.0", Some(44839)),
     (unstable, avx512_target_feature, "1.27.0", Some(44839)),
@@ -317,7 +317,7 @@ declare_features! (
     (unstable, prfchw_target_feature, "1.78.0", Some(44839)),
     (unstable, riscv_target_feature, "1.45.0", Some(44839)),
     (unstable, rtm_target_feature, "1.35.0", Some(44839)),
-    (unstable, s390x_target_feature, "CURRENT_RUSTC_VERSION", Some(44839)),
+    (unstable, s390x_target_feature, "1.82.0", Some(44839)),
     (unstable, sse4a_target_feature, "1.27.0", Some(44839)),
     (unstable, tbm_target_feature, "1.27.0", Some(44839)),
     (unstable, wasm_target_feature, "1.30.0", Some(44839)),
@@ -474,7 +474,7 @@ declare_features! (
     /// Allows the use of `#[ffi_pure]` on foreign functions.
     (unstable, ffi_pure, "1.45.0", Some(58329)),
     /// Controlling the behavior of fmt::Debug
-    (unstable, fmt_debug, "CURRENT_RUSTC_VERSION", Some(129709)),
+    (unstable, fmt_debug, "1.82.0", Some(129709)),
     /// Allows using `#[repr(align(...))]` on function items
     (unstable, fn_align, "1.53.0", Some(82232)),
     /// Support delegating implementation of functions to other already implemented functions.
@@ -586,8 +586,8 @@ declare_features! (
     /// Allows `extern "rust-cold"`.
     (unstable, rust_cold_cc, "1.63.0", Some(97544)),
     /// Allows use of x86 SHA512, SM3 and SM4 target-features and intrinsics
-    (unstable, sha512_sm_x86, "CURRENT_RUSTC_VERSION", Some(126624)),
-    /// Shorten the tail expression lifetime
+    (unstable, sha512_sm_x86, "1.82.0", Some(126624)),
+    /// Shortern the tail expression lifetime
     (unstable, shorter_tail_lifetimes, "1.79.0", Some(123739)),
     /// Allows the use of SIMD types in functions declared in `extern` blocks.
     (unstable, simd_ffi, "1.0.0", Some(27731)),
@@ -625,7 +625,7 @@ declare_features! (
     (incomplete, unnamed_fields, "1.74.0", Some(49804)),
     /// Allows const generic parameters to be defined with types that
     /// are not `Sized`, e.g. `fn foo<const N: [u8]>() {`.
-    (incomplete, unsized_const_params, "CURRENT_RUSTC_VERSION", Some(95174)),
+    (incomplete, unsized_const_params, "1.82.0", Some(95174)),
     /// Allows unsized fn parameters.
     (internal, unsized_fn_params, "1.49.0", Some(48055)),
     /// Allows unsized rvalues at arguments and parameters.
diff --git a/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs b/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs
index bd8b43e28e5..9fce927e2c7 100644
--- a/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs
+++ b/compiler/rustc_hir_analysis/src/coherence/inherent_impls.rs
@@ -93,7 +93,8 @@ impl<'tcx> InherentCollect<'tcx> {
                 }
             }
 
-            if let Some(simp) = simplify_type(self.tcx, self_ty, TreatParams::AsCandidateKey) {
+            if let Some(simp) = simplify_type(self.tcx, self_ty, TreatParams::InstantiateWithInfer)
+            {
                 self.impls_map.incoherent_impls.entry(simp).or_default().push(impl_def_id);
             } else {
                 bug!("unexpected self type: {:?}", self_ty);
@@ -132,7 +133,7 @@ impl<'tcx> InherentCollect<'tcx> {
             }
         }
 
-        if let Some(simp) = simplify_type(self.tcx, ty, TreatParams::AsCandidateKey) {
+        if let Some(simp) = simplify_type(self.tcx, ty, TreatParams::InstantiateWithInfer) {
             self.impls_map.incoherent_impls.entry(simp).or_default().push(impl_def_id);
         } else {
             bug!("unexpected primitive type: {:?}", ty);
diff --git a/compiler/rustc_hir_analysis/src/collect.rs b/compiler/rustc_hir_analysis/src/collect.rs
index 92111805ab4..ac9976148e2 100644
--- a/compiler/rustc_hir_analysis/src/collect.rs
+++ b/compiler/rustc_hir_analysis/src/collect.rs
@@ -1698,8 +1698,6 @@ fn impl_trait_header(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option<ty::ImplTrai
             trait_ref: ty::EarlyBinder::bind(trait_ref),
             safety: impl_.safety,
             polarity: polarity_of_impl(tcx, def_id, impl_, item.span),
-            do_not_recommend: tcx.features().do_not_recommend
-                && tcx.has_attrs_with_path(def_id, &[sym::diagnostic, sym::do_not_recommend]),
         }
     })
 }
diff --git a/compiler/rustc_hir_analysis/src/collect/generics_of.rs b/compiler/rustc_hir_analysis/src/collect/generics_of.rs
index be6c3f048dc..ba01ea3f512 100644
--- a/compiler/rustc_hir_analysis/src/collect/generics_of.rs
+++ b/compiler/rustc_hir_analysis/src/collect/generics_of.rs
@@ -59,6 +59,12 @@ pub(super) fn generics_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Generics {
     let hir_id = tcx.local_def_id_to_hir_id(def_id);
 
     let node = tcx.hir_node(hir_id);
+    if let Some(sig) = node.fn_sig()
+        && let Some(sig_id) = sig.decl.opt_delegation_sig_id()
+    {
+        return inherit_generics_for_delegation_item(tcx, def_id, sig_id);
+    }
+
     let parent_def_id = match node {
         Node::ImplItem(_)
         | Node::TraitItem(_)
@@ -229,16 +235,6 @@ pub(super) fn generics_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Generics {
                 // inherit the generics of the item.
                 Some(parent.to_def_id())
             }
-            ItemKind::Fn(sig, _, _) => {
-                // For a delegation item inherit generics from callee.
-                if let Some(sig_id) = sig.decl.opt_delegation_sig_id()
-                    && let Some(generics) =
-                        inherit_generics_for_delegation_item(tcx, def_id, sig_id)
-                {
-                    return generics;
-                }
-                None
-            }
             _ => None,
         },
         _ => None,
diff --git a/compiler/rustc_hir_analysis/src/collect/predicates_of.rs b/compiler/rustc_hir_analysis/src/collect/predicates_of.rs
index a47aaf25e95..7243e85ce98 100644
--- a/compiler/rustc_hir_analysis/src/collect/predicates_of.rs
+++ b/compiler/rustc_hir_analysis/src/collect/predicates_of.rs
@@ -138,6 +138,12 @@ fn gather_explicit_predicates_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Gen
     let hir_id = tcx.local_def_id_to_hir_id(def_id);
     let node = tcx.hir_node(hir_id);
 
+    if let Some(sig) = node.fn_sig()
+        && let Some(sig_id) = sig.decl.opt_delegation_sig_id()
+    {
+        return inherit_predicates_for_delegation_item(tcx, def_id, sig_id);
+    }
+
     let mut is_trait = None;
     let mut is_default_impl_trait = None;
 
@@ -164,16 +170,6 @@ fn gather_explicit_predicates_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Gen
             ItemKind::Trait(_, _, _, self_bounds, ..) | ItemKind::TraitAlias(_, self_bounds) => {
                 is_trait = Some(self_bounds);
             }
-
-            ItemKind::Fn(sig, _, _) => {
-                // For a delegation item inherit predicates from callee.
-                if let Some(sig_id) = sig.decl.opt_delegation_sig_id()
-                    && let Some(predicates) =
-                        inherit_predicates_for_delegation_item(tcx, def_id, sig_id)
-                {
-                    return predicates;
-                }
-            }
             _ => {}
         }
     };
diff --git a/compiler/rustc_hir_analysis/src/delegation.rs b/compiler/rustc_hir_analysis/src/delegation.rs
index 20aaa43219f..2c9f20b7840 100644
--- a/compiler/rustc_hir_analysis/src/delegation.rs
+++ b/compiler/rustc_hir_analysis/src/delegation.rs
@@ -1,3 +1,7 @@
+//! Support inheriting generic parameters and predicates for function delegation.
+//!
+//! For more information about delegation design, see the tracking issue #118212.
+
 use std::assert_matches::debug_assert_matches;
 
 use rustc_data_structures::fx::FxHashMap;
@@ -5,7 +9,7 @@ use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_middle::ty::fold::{TypeFoldable, TypeFolder, TypeSuperFoldable};
 use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_span::ErrorGuaranteed;
+use rustc_span::{ErrorGuaranteed, Span};
 use rustc_type_ir::visit::TypeVisitableExt;
 
 type RemapTable = FxHashMap<u32, u32>;
@@ -76,127 +80,381 @@ fn fn_kind<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> FnKind {
     }
 }
 
+/// Given the current context(caller and callee `FnKind`), it specifies
+/// the policy of predicates and generic parameters inheritance.
+#[derive(Clone, Copy, Debug, PartialEq)]
+enum InheritanceKind {
+    /// Copying all predicates and parameters, including those of the parent
+    /// container.
+    ///
+    /// Boolean value defines whether the `Self` parameter or `Self: Trait`
+    /// predicate are copied. It's always equal to `false` except when
+    /// delegating from a free function to a trait method.
+    ///
+    /// FIXME(fn_delegation): This often leads to type inference
+    /// errors. Support providing generic arguments or restrict use sites.
+    WithParent(bool),
+    /// The trait implementation should be compatible with the original trait.
+    /// Therefore, for trait implementations only the method's own parameters
+    /// and predicates are copied.
+    Own,
+}
+
+struct GenericsBuilder<'tcx> {
+    tcx: TyCtxt<'tcx>,
+    sig_id: DefId,
+    parent: Option<DefId>,
+    inh_kind: InheritanceKind,
+}
+
+impl<'tcx> GenericsBuilder<'tcx> {
+    fn new(tcx: TyCtxt<'tcx>, sig_id: DefId) -> GenericsBuilder<'tcx> {
+        GenericsBuilder { tcx, sig_id, parent: None, inh_kind: InheritanceKind::WithParent(false) }
+    }
+
+    fn with_parent(mut self, parent: DefId) -> Self {
+        self.parent = Some(parent);
+        self
+    }
+
+    fn with_inheritance_kind(mut self, inh_kind: InheritanceKind) -> Self {
+        self.inh_kind = inh_kind;
+        self
+    }
+
+    fn build(self) -> ty::Generics {
+        let mut own_params = vec![];
+
+        let sig_generics = self.tcx.generics_of(self.sig_id);
+        if let InheritanceKind::WithParent(has_self) = self.inh_kind
+            && let Some(parent_def_id) = sig_generics.parent
+        {
+            let sig_parent_generics = self.tcx.generics_of(parent_def_id);
+            own_params.append(&mut sig_parent_generics.own_params.clone());
+            if !has_self {
+                own_params.remove(0);
+            }
+        }
+        own_params.append(&mut sig_generics.own_params.clone());
+
+        // Lifetime parameters must be declared before type and const parameters.
+        // Therefore, When delegating from a free function to a associated function,
+        // generic parameters need to be reordered:
+        //
+        // trait Trait<'a, A> {
+        //     fn foo<'b, B>(...) {...}
+        // }
+        //
+        // reuse Trait::foo;
+        // desugaring:
+        // fn foo<'a, 'b, This: Trait<'a, A>, A, B>(...) {
+        //     Trait::foo(...)
+        // }
+        own_params.sort_by_key(|key| key.kind.is_ty_or_const());
+
+        let param_def_id_to_index =
+            own_params.iter().map(|param| (param.def_id, param.index)).collect();
+
+        let (parent_count, has_self) = if let Some(def_id) = self.parent {
+            let parent_generics = self.tcx.generics_of(def_id);
+            let parent_kind = self.tcx.def_kind(def_id);
+            (parent_generics.count(), parent_kind == DefKind::Trait)
+        } else {
+            (0, false)
+        };
+
+        for (idx, param) in own_params.iter_mut().enumerate() {
+            param.index = (idx + parent_count) as u32;
+            // FIXME(fn_delegation): Default parameters are not inherited, because they are
+            // not permitted in functions. Therefore, there are 2 options here:
+            //
+            // - We can create non-default generic parameters.
+            // - We can substitute default parameters into the signature.
+            //
+            // At the moment, first option has been selected as the most general.
+            if let ty::GenericParamDefKind::Type { has_default, .. }
+            | ty::GenericParamDefKind::Const { has_default, .. } = &mut param.kind
+            {
+                *has_default = false;
+            }
+        }
+
+        ty::Generics {
+            parent: self.parent,
+            parent_count,
+            own_params,
+            param_def_id_to_index,
+            has_self,
+            has_late_bound_regions: sig_generics.has_late_bound_regions,
+            host_effect_index: sig_generics.host_effect_index,
+        }
+    }
+}
+
+struct PredicatesBuilder<'tcx> {
+    tcx: TyCtxt<'tcx>,
+    sig_id: DefId,
+    parent: Option<DefId>,
+    inh_kind: InheritanceKind,
+    args: ty::GenericArgsRef<'tcx>,
+}
+
+impl<'tcx> PredicatesBuilder<'tcx> {
+    fn new(
+        tcx: TyCtxt<'tcx>,
+        args: ty::GenericArgsRef<'tcx>,
+        sig_id: DefId,
+    ) -> PredicatesBuilder<'tcx> {
+        PredicatesBuilder {
+            tcx,
+            sig_id,
+            parent: None,
+            inh_kind: InheritanceKind::WithParent(false),
+            args,
+        }
+    }
+
+    fn with_parent(mut self, parent: DefId) -> Self {
+        self.parent = Some(parent);
+        self
+    }
+
+    fn with_inheritance_kind(mut self, inh_kind: InheritanceKind) -> Self {
+        self.inh_kind = inh_kind;
+        self
+    }
+
+    fn build(self) -> ty::GenericPredicates<'tcx> {
+        struct PredicatesCollector<'tcx> {
+            tcx: TyCtxt<'tcx>,
+            preds: Vec<(ty::Clause<'tcx>, Span)>,
+            args: ty::GenericArgsRef<'tcx>,
+        }
+
+        impl<'tcx> PredicatesCollector<'tcx> {
+            fn new(tcx: TyCtxt<'tcx>, args: ty::GenericArgsRef<'tcx>) -> PredicatesCollector<'tcx> {
+                PredicatesCollector { tcx, preds: vec![], args }
+            }
+
+            fn with_own_preds(
+                mut self,
+                f: impl Fn(DefId) -> ty::GenericPredicates<'tcx>,
+                def_id: DefId,
+            ) -> Self {
+                let preds = f(def_id).instantiate_own(self.tcx, self.args);
+                self.preds.extend(preds);
+                self
+            }
+
+            fn with_preds(
+                mut self,
+                f: impl Fn(DefId) -> ty::GenericPredicates<'tcx> + Copy,
+                def_id: DefId,
+            ) -> Self {
+                let preds = f(def_id);
+                if let Some(parent_def_id) = preds.parent {
+                    self = self.with_own_preds(f, parent_def_id);
+                }
+                self.with_own_preds(f, def_id)
+            }
+        }
+        let collector = PredicatesCollector::new(self.tcx, self.args);
+
+        // `explicit_predicates_of` is used here to avoid copying `Self: Trait` predicate.
+        // Note: `predicates_of` query can also add inferred outlives predicates, but that
+        // is not the case here as `sig_id` is either a trait or a function.
+        let preds = match self.inh_kind {
+            InheritanceKind::WithParent(false) => {
+                collector.with_preds(|def_id| self.tcx.explicit_predicates_of(def_id), self.sig_id)
+            }
+            InheritanceKind::WithParent(true) => {
+                collector.with_preds(|def_id| self.tcx.predicates_of(def_id), self.sig_id)
+            }
+            InheritanceKind::Own => {
+                collector.with_own_preds(|def_id| self.tcx.predicates_of(def_id), self.sig_id)
+            }
+        }
+        .preds;
+
+        ty::GenericPredicates {
+            parent: self.parent,
+            predicates: self.tcx.arena.alloc_from_iter(preds),
+            // FIXME(fn_delegation): Support effects.
+            effects_min_tys: ty::List::empty(),
+        }
+    }
+}
+
+struct GenericArgsBuilder<'tcx> {
+    tcx: TyCtxt<'tcx>,
+    remap_table: RemapTable,
+    sig_id: DefId,
+    def_id: LocalDefId,
+}
+
+impl<'tcx> GenericArgsBuilder<'tcx> {
+    fn new(tcx: TyCtxt<'tcx>, sig_id: DefId, def_id: LocalDefId) -> GenericArgsBuilder<'tcx> {
+        GenericArgsBuilder { tcx, remap_table: FxHashMap::default(), sig_id, def_id }
+    }
+
+    fn build_from_args(mut self, args: ty::GenericArgsRef<'tcx>) -> ty::GenericArgsRef<'tcx> {
+        let caller_generics = self.tcx.generics_of(self.def_id);
+        let callee_generics = self.tcx.generics_of(self.sig_id);
+
+        for caller_param in &caller_generics.own_params {
+            let callee_index =
+                callee_generics.param_def_id_to_index(self.tcx, caller_param.def_id).unwrap();
+            self.remap_table.insert(callee_index, caller_param.index);
+        }
+
+        let mut folder = ParamIndexRemapper { tcx: self.tcx, remap_table: self.remap_table };
+        args.fold_with(&mut folder)
+    }
+}
+
 fn create_generic_args<'tcx>(
     tcx: TyCtxt<'tcx>,
     def_id: LocalDefId,
     sig_id: DefId,
 ) -> ty::GenericArgsRef<'tcx> {
-    let caller_generics = tcx.generics_of(def_id);
-    let callee_generics = tcx.generics_of(sig_id);
+    let builder = GenericArgsBuilder::new(tcx, sig_id, def_id);
 
     let caller_kind = fn_kind(tcx, def_id.into());
     let callee_kind = fn_kind(tcx, sig_id);
-    // FIXME(fn_delegation): Support generics on associated delegation items.
-    // Error will be reported in `check_constraints`.
     match (caller_kind, callee_kind) {
-        (FnKind::Free, _) => {
-            // Lifetime parameters must be declared before type and const parameters.
-            // Therefore, When delegating from a free function to a associated function,
-            // generic parameters need to be reordered:
-            //
-            // trait Trait<'a, A> {
-            //     fn foo<'b, B>(...) {...}
-            // }
-            //
-            // reuse Trait::foo;
-            // desugaring:
-            // fn foo<'a, 'b, This: Trait<'a, A>, A, B>(...) {
-            //     Trait::foo(...)
-            // }
-            let mut remap_table = RemapTable::default();
-            for caller_param in &caller_generics.own_params {
-                let callee_index =
-                    callee_generics.param_def_id_to_index(tcx, caller_param.def_id).unwrap();
-                remap_table.insert(callee_index, caller_param.index);
-            }
-            let mut folder = ParamIndexRemapper { tcx, remap_table };
-            ty::GenericArgs::identity_for_item(tcx, sig_id).fold_with(&mut folder)
+        (FnKind::Free, FnKind::Free)
+        | (FnKind::Free, FnKind::AssocTrait)
+        | (FnKind::AssocInherentImpl, FnKind::Free)
+        | (FnKind::AssocTrait, FnKind::Free)
+        | (FnKind::AssocTrait, FnKind::AssocTrait) => {
+            let args = ty::GenericArgs::identity_for_item(tcx, sig_id);
+            builder.build_from_args(args)
         }
-        // FIXME(fn_delegation): Only `Self` param supported here.
-        (FnKind::AssocTraitImpl, FnKind::AssocTrait)
-        | (FnKind::AssocInherentImpl, FnKind::AssocTrait) => {
+
+        (FnKind::AssocTraitImpl, FnKind::AssocTrait) => {
+            let callee_generics = tcx.generics_of(sig_id);
+            let parent = tcx.parent(def_id.into());
+            let parent_args =
+                tcx.impl_trait_header(parent).unwrap().trait_ref.instantiate_identity().args;
+
+            let trait_args = ty::GenericArgs::identity_for_item(tcx, sig_id);
+            let method_args = tcx.mk_args_from_iter(trait_args.iter().skip(callee_generics.parent_count));
+            let method_args = builder.build_from_args(method_args);
+
+            tcx.mk_args_from_iter(parent_args.iter().chain(method_args))
+        }
+
+        (FnKind::AssocInherentImpl, FnKind::AssocTrait) => {
             let parent = tcx.parent(def_id.into());
             let self_ty = tcx.type_of(parent).instantiate_identity();
             let generic_self_ty = ty::GenericArg::from(self_ty);
-            tcx.mk_args_from_iter(std::iter::once(generic_self_ty))
+
+            let trait_args = ty::GenericArgs::identity_for_item(tcx, sig_id);
+            let trait_args = builder.build_from_args(trait_args);
+
+            let args = std::iter::once(generic_self_ty).chain(trait_args.iter().skip(1));
+            tcx.mk_args_from_iter(args)
         }
-        _ => ty::GenericArgs::identity_for_item(tcx, sig_id),
+
+        // For trait impl's `sig_id` is always equal to the corresponding trait method.
+        (FnKind::AssocTraitImpl, _)
+        | (_, FnKind::AssocTraitImpl)
+        // Delegation to inherent methods is not yet supported.
+        | (_, FnKind::AssocInherentImpl) => unreachable!(),
     }
 }
 
+// FIXME(fn_delegation): Move generics inheritance to the AST->HIR lowering.
+// For now, generic parameters are not propagated to the generated call,
+// which leads to inference errors:
+//
+// fn foo<T>(x: i32) {}
+//
+// reuse foo as bar;
+// desugaring:
+// fn bar<T>() {
+//   foo::<_>() // ERROR: type annotations needed
+// }
 pub(crate) fn inherit_generics_for_delegation_item<'tcx>(
     tcx: TyCtxt<'tcx>,
     def_id: LocalDefId,
     sig_id: DefId,
-) -> Option<ty::Generics> {
-    // FIXME(fn_delegation): Support generics on associated delegation items.
-    // Error will be reported in `check_constraints`.
-    if fn_kind(tcx, def_id.into()) != FnKind::Free {
-        return None;
-    }
+) -> ty::Generics {
+    let builder = GenericsBuilder::new(tcx, sig_id);
 
-    let mut own_params = vec![];
-
-    let callee_generics = tcx.generics_of(sig_id);
-    if let Some(parent_sig_id) = callee_generics.parent {
-        let parent_sig_generics = tcx.generics_of(parent_sig_id);
-        own_params.append(&mut parent_sig_generics.own_params.clone());
-    }
-    own_params.append(&mut callee_generics.own_params.clone());
+    let caller_kind = fn_kind(tcx, def_id.into());
+    let callee_kind = fn_kind(tcx, sig_id);
+    match (caller_kind, callee_kind) {
+        (FnKind::Free, FnKind::Free)
+        | (FnKind::Free, FnKind::AssocTrait) => builder.with_inheritance_kind(InheritanceKind::WithParent(true)).build(),
+
+        (FnKind::AssocTraitImpl, FnKind::AssocTrait) => {
+            builder
+            .with_parent(tcx.parent(def_id.into()))
+            .with_inheritance_kind(InheritanceKind::Own)
+            .build()
+        }
 
-    // Lifetimes go first.
-    own_params.sort_by_key(|key| key.kind.is_ty_or_const());
+        (FnKind::AssocInherentImpl, FnKind::AssocTrait)
+        | (FnKind::AssocTrait, FnKind::AssocTrait) => {
+            builder
+            .with_parent(tcx.parent(def_id.into()))
+            .build()
+        }
 
-    for (idx, param) in own_params.iter_mut().enumerate() {
-        param.index = idx as u32;
-        // Default parameters are not inherited: they are not allowed
-        // in fn's.
-        if let ty::GenericParamDefKind::Type { has_default, .. }
-        | ty::GenericParamDefKind::Const { has_default, .. } = &mut param.kind
-        {
-            *has_default = false;
+        (FnKind::AssocInherentImpl, FnKind::Free)
+        | (FnKind::AssocTrait, FnKind::Free) => {
+            builder
+            .with_parent(tcx.parent(def_id.into()))
+            .build()
         }
-    }
 
-    let param_def_id_to_index =
-        own_params.iter().map(|param| (param.def_id, param.index)).collect();
-
-    Some(ty::Generics {
-        parent: None,
-        parent_count: 0,
-        own_params,
-        param_def_id_to_index,
-        has_self: false,
-        has_late_bound_regions: callee_generics.has_late_bound_regions,
-        host_effect_index: callee_generics.host_effect_index,
-    })
+        // For trait impl's `sig_id` is always equal to the corresponding trait method.
+        (FnKind::AssocTraitImpl, _)
+        | (_, FnKind::AssocTraitImpl)
+        // Delegation to inherent methods is not yet supported.
+        | (_, FnKind::AssocInherentImpl) => unreachable!(),
+    }
 }
 
 pub(crate) fn inherit_predicates_for_delegation_item<'tcx>(
     tcx: TyCtxt<'tcx>,
     def_id: LocalDefId,
     sig_id: DefId,
-) -> Option<ty::GenericPredicates<'tcx>> {
-    // FIXME(fn_delegation): Support generics on associated delegation items.
-    // Error will be reported in `check_constraints`.
-    if fn_kind(tcx, def_id.into()) != FnKind::Free {
-        return None;
-    }
-
-    let callee_predicates = tcx.predicates_of(sig_id);
+) -> ty::GenericPredicates<'tcx> {
     let args = create_generic_args(tcx, def_id, sig_id);
+    let builder = PredicatesBuilder::new(tcx, args, sig_id);
 
-    let mut preds = vec![];
-    if let Some(parent_id) = callee_predicates.parent {
-        preds.extend(tcx.predicates_of(parent_id).instantiate_own(tcx, args));
-    }
-    preds.extend(callee_predicates.instantiate_own(tcx, args));
+    let caller_kind = fn_kind(tcx, def_id.into());
+    let callee_kind = fn_kind(tcx, sig_id);
+    match (caller_kind, callee_kind) {
+        (FnKind::Free, FnKind::Free)
+        | (FnKind::Free, FnKind::AssocTrait) => {
+            builder.with_inheritance_kind(InheritanceKind::WithParent(true)).build()
+        }
+
+        (FnKind::AssocTraitImpl, FnKind::AssocTrait) => {
+            builder
+            .with_parent(tcx.parent(def_id.into()))
+            .with_inheritance_kind(InheritanceKind::Own)
+            .build()
+        }
+
+        (FnKind::AssocInherentImpl, FnKind::AssocTrait)
+        | (FnKind::AssocTrait, FnKind::AssocTrait)
+        | (FnKind::AssocInherentImpl, FnKind::Free)
+        | (FnKind::AssocTrait, FnKind::Free) => {
+            builder
+                .with_parent(tcx.parent(def_id.into()))
+                .build()
+        }
 
-    Some(ty::GenericPredicates {
-        parent: None,
-        predicates: tcx.arena.alloc_from_iter(preds),
-        effects_min_tys: ty::List::empty(),
-    })
+        // For trait impl's `sig_id` is always equal to the corresponding trait method.
+        (FnKind::AssocTraitImpl, _)
+        | (_, FnKind::AssocTraitImpl)
+        // Delegation to inherent methods is not yet supported.
+        | (_, FnKind::AssocInherentImpl) => unreachable!(),
+    }
 }
 
 fn check_constraints<'tcx>(
@@ -224,19 +482,6 @@ fn check_constraints<'tcx>(
         emit("recursive delegation is not supported yet");
     }
 
-    if fn_kind(tcx, def_id.into()) != FnKind::Free {
-        let sig_generics = tcx.generics_of(sig_id);
-        let parent = tcx.parent(def_id.into());
-        let parent_generics = tcx.generics_of(parent);
-
-        let parent_has_self = parent_generics.has_self as usize;
-        let sig_has_self = sig_generics.has_self as usize;
-
-        if sig_generics.count() > sig_has_self || parent_generics.count() > parent_has_self {
-            emit("early bound generics are not supported for associated delegation items");
-        }
-    }
-
     ret
 }
 
diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/cmse.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/cmse.rs
index 1b73cecd666..5150db7f51b 100644
--- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/cmse.rs
+++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/cmse.rs
@@ -72,8 +72,11 @@ fn is_valid_cmse_inputs<'tcx>(
     let mut span = None;
     let mut accum = 0u64;
 
-    for (index, arg_def) in fn_sig.inputs().iter().enumerate() {
-        let layout = tcx.layout_of(ParamEnv::reveal_all().and(*arg_def.skip_binder()))?;
+    // this type is only used for layout computation, which does not rely on regions
+    let fn_sig = tcx.instantiate_bound_regions_with_erased(fn_sig);
+
+    for (index, ty) in fn_sig.inputs().iter().enumerate() {
+        let layout = tcx.layout_of(ParamEnv::reveal_all().and(*ty))?;
 
         let align = layout.layout.align().abi.bytes();
         let size = layout.layout.size().bytes();
@@ -98,7 +101,10 @@ fn is_valid_cmse_output<'tcx>(
     tcx: TyCtxt<'tcx>,
     fn_sig: ty::PolyFnSig<'tcx>,
 ) -> Result<bool, &'tcx LayoutError<'tcx>> {
-    let mut ret_ty = fn_sig.output().skip_binder();
+    // this type is only used for layout computation, which does not rely on regions
+    let fn_sig = tcx.instantiate_bound_regions_with_erased(fn_sig);
+
+    let mut ret_ty = fn_sig.output();
     let layout = tcx.layout_of(ParamEnv::reveal_all().and(ret_ty))?;
     let size = layout.layout.size().bytes();
 
diff --git a/compiler/rustc_hir_typeck/src/expr.rs b/compiler/rustc_hir_typeck/src/expr.rs
index f0d47e584ac..9bad5633b69 100644
--- a/compiler/rustc_hir_typeck/src/expr.rs
+++ b/compiler/rustc_hir_typeck/src/expr.rs
@@ -447,7 +447,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 // this time with enough precision to check that the value
                 // whose address was taken can actually be made to live as long
                 // as it needs to live.
-                let region = self.next_region_var(infer::AddrOfRegion(expr.span));
+                let region = self.next_region_var(infer::BorrowRegion(expr.span));
                 Ty::new_ref(self.tcx, region, ty, mutbl)
             }
         }
diff --git a/compiler/rustc_hir_typeck/src/method/prelude_edition_lints.rs b/compiler/rustc_hir_typeck/src/method/prelude_edition_lints.rs
index 0790c6f9a59..ac5e1040803 100644
--- a/compiler/rustc_hir_typeck/src/method/prelude_edition_lints.rs
+++ b/compiler/rustc_hir_typeck/src/method/prelude_edition_lints.rs
@@ -63,8 +63,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     // Instead, the problem is that the array-into_iter hack will no longer
                     // apply in Rust 2021.
                     (ARRAY_INTO_ITER, "2021")
-                } else if self_ty.is_box()
-                    && self_ty.boxed_ty().is_slice()
+                } else if self_ty.boxed_ty().is_some_and(Ty::is_slice)
                     && !span.at_least_rust_2024()
                 {
                     // In this case, it wasn't really a prelude addition that was the problem.
diff --git a/compiler/rustc_hir_typeck/src/method/probe.rs b/compiler/rustc_hir_typeck/src/method/probe.rs
index 2fdba8446bd..3ba3429cbb3 100644
--- a/compiler/rustc_hir_typeck/src/method/probe.rs
+++ b/compiler/rustc_hir_typeck/src/method/probe.rs
@@ -715,7 +715,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
     }
 
     fn assemble_inherent_candidates_for_incoherent_ty(&mut self, self_ty: Ty<'tcx>) {
-        let Some(simp) = simplify_type(self.tcx, self_ty, TreatParams::AsCandidateKey) else {
+        let Some(simp) = simplify_type(self.tcx, self_ty, TreatParams::InstantiateWithInfer) else {
             bug!("unexpected incoherent type: {:?}", self_ty)
         };
         for &impl_def_id in self.tcx.incoherent_impls(simp).into_iter().flatten() {
@@ -1485,8 +1485,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
 
                         // Some trait methods are excluded for boxed slices before 2024.
                         // (`boxed_slice.into_iter()` wants a slice iterator for compatibility.)
-                        if self_ty.is_box()
-                            && self_ty.boxed_ty().is_slice()
+                        if self_ty.boxed_ty().is_some_and(Ty::is_slice)
                             && !method_name.span.at_least_rust_2024()
                         {
                             let trait_def = self.tcx.trait_def(poly_trait_ref.def_id());
diff --git a/compiler/rustc_hir_typeck/src/method/suggest.rs b/compiler/rustc_hir_typeck/src/method/suggest.rs
index 9ea57e4aa61..14ad5830111 100644
--- a/compiler/rustc_hir_typeck/src/method/suggest.rs
+++ b/compiler/rustc_hir_typeck/src/method/suggest.rs
@@ -2235,8 +2235,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             let target_ty = self
                 .autoderef(sugg_span, rcvr_ty)
                 .find(|(rcvr_ty, _)| {
-                    DeepRejectCtxt::new(self.tcx, TreatParams::ForLookup)
-                        .types_may_unify(*rcvr_ty, impl_ty)
+                    DeepRejectCtxt::relate_rigid_infer(self.tcx).types_may_unify(*rcvr_ty, impl_ty)
                 })
                 .map_or(impl_ty, |(ty, _)| ty)
                 .peel_refs();
@@ -2498,7 +2497,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             .into_iter()
             .any(|info| self.associated_value(info.def_id, item_name).is_some());
         let found_assoc = |ty: Ty<'tcx>| {
-            simplify_type(tcx, ty, TreatParams::AsCandidateKey)
+            simplify_type(tcx, ty, TreatParams::InstantiateWithInfer)
                 .and_then(|simp| {
                     tcx.incoherent_impls(simp)
                         .into_iter()
@@ -3963,7 +3962,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 // cases where a positive bound implies a negative impl.
                 (candidates, Vec::new())
             } else if let Some(simp_rcvr_ty) =
-                simplify_type(self.tcx, rcvr_ty, TreatParams::ForLookup)
+                simplify_type(self.tcx, rcvr_ty, TreatParams::AsRigid)
             {
                 let mut potential_candidates = Vec::new();
                 let mut explicitly_negative = Vec::new();
@@ -3981,7 +3980,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         .any(|header| {
                             let imp = header.trait_ref.instantiate_identity();
                             let imp_simp =
-                                simplify_type(self.tcx, imp.self_ty(), TreatParams::ForLookup);
+                                simplify_type(self.tcx, imp.self_ty(), TreatParams::AsRigid);
                             imp_simp.is_some_and(|s| s == simp_rcvr_ty)
                         })
                     {
diff --git a/compiler/rustc_hir_typeck/src/upvar.rs b/compiler/rustc_hir_typeck/src/upvar.rs
index 5789e60ebbe..7688a63a30a 100644
--- a/compiler/rustc_hir_typeck/src/upvar.rs
+++ b/compiler/rustc_hir_typeck/src/upvar.rs
@@ -39,7 +39,6 @@ use rustc_hir as hir;
 use rustc_hir::def_id::LocalDefId;
 use rustc_hir::intravisit::{self, Visitor};
 use rustc_hir::HirId;
-use rustc_infer::infer::UpvarRegion;
 use rustc_middle::hir::place::{Place, PlaceBase, PlaceWithHirId, Projection, ProjectionKind};
 use rustc_middle::mir::FakeReadCause;
 use rustc_middle::traits::ObligationCauseCode;
@@ -425,7 +424,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             self.tcx,
                             upvar_ty,
                             capture,
-                            if needs_ref { Some(closure_env_region) } else { child_capture.region },
+                            if needs_ref {
+                                closure_env_region
+                            } else {
+                                self.tcx.lifetimes.re_erased
+                            },
                         );
                     },
                 ),
@@ -587,7 +590,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
                 debug!(?captured_place.place, ?upvar_ty, ?capture, ?captured_place.mutability);
 
-                apply_capture_kind_on_capture_ty(self.tcx, upvar_ty, capture, captured_place.region)
+                apply_capture_kind_on_capture_ty(
+                    self.tcx,
+                    upvar_ty,
+                    capture,
+                    self.tcx.lifetimes.re_erased,
+                )
             })
             .collect()
     }
@@ -775,13 +783,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
             let Some(min_cap_list) = root_var_min_capture_list.get_mut(&var_hir_id) else {
                 let mutability = self.determine_capture_mutability(&typeck_results, &place);
-                let min_cap_list = vec![ty::CapturedPlace {
-                    var_ident,
-                    place,
-                    info: capture_info,
-                    mutability,
-                    region: None,
-                }];
+                let min_cap_list =
+                    vec![ty::CapturedPlace { var_ident, place, info: capture_info, mutability }];
                 root_var_min_capture_list.insert(var_hir_id, min_cap_list);
                 continue;
             };
@@ -874,34 +877,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // Only need to insert when we don't have an ancestor in the existing min capture list
             if !ancestor_found {
                 let mutability = self.determine_capture_mutability(&typeck_results, &place);
-                let captured_place = ty::CapturedPlace {
-                    var_ident,
-                    place,
-                    info: updated_capture_info,
-                    mutability,
-                    region: None,
-                };
+                let captured_place =
+                    ty::CapturedPlace { var_ident, place, info: updated_capture_info, mutability };
                 min_cap_list.push(captured_place);
             }
         }
 
-        // For each capture that is determined to be captured by ref, add region info.
-        for (_, captures) in &mut root_var_min_capture_list {
-            for capture in captures {
-                match capture.info.capture_kind {
-                    ty::UpvarCapture::ByRef(_) => {
-                        let PlaceBase::Upvar(upvar_id) = capture.place.base else {
-                            bug!("expected upvar")
-                        };
-                        let origin = UpvarRegion(upvar_id, closure_span);
-                        let upvar_region = self.next_region_var(origin);
-                        capture.region = Some(upvar_region);
-                    }
-                    _ => (),
-                }
-            }
-        }
-
         debug!(
             "For closure={:?}, min_captures before sorting={:?}",
             closure_def_id, root_var_min_capture_list
@@ -1195,7 +1176,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     self.tcx,
                     ty,
                     max_capture_info.capture_kind,
-                    Some(self.tcx.lifetimes.re_erased),
+                    self.tcx.lifetimes.re_erased,
                 )
             }
         };
@@ -1217,7 +1198,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 self.tcx,
                 capture.place.ty(),
                 capture.info.capture_kind,
-                Some(self.tcx.lifetimes.re_erased),
+                self.tcx.lifetimes.re_erased,
             );
 
             // Checks if a capture implements any of the auto traits
@@ -1935,13 +1916,11 @@ fn apply_capture_kind_on_capture_ty<'tcx>(
     tcx: TyCtxt<'tcx>,
     ty: Ty<'tcx>,
     capture_kind: UpvarCapture,
-    region: Option<ty::Region<'tcx>>,
+    region: ty::Region<'tcx>,
 ) -> Ty<'tcx> {
     match capture_kind {
         ty::UpvarCapture::ByValue => ty,
-        ty::UpvarCapture::ByRef(kind) => {
-            Ty::new_ref(tcx, region.unwrap(), ty, kind.to_mutbl_lossy())
-        }
+        ty::UpvarCapture::ByRef(kind) => Ty::new_ref(tcx, region, ty, kind.to_mutbl_lossy()),
     }
 }
 
diff --git a/compiler/rustc_index/src/interval.rs b/compiler/rustc_index/src/interval.rs
index be028feca60..34f541a8cc6 100644
--- a/compiler/rustc_index/src/interval.rs
+++ b/compiler/rustc_index/src/interval.rs
@@ -17,8 +17,8 @@ mod tests;
 /// first value of the following element.
 #[derive(Debug, Clone)]
 pub struct IntervalSet<I> {
-    // Start, end
-    map: SmallVec<[(u32, u32); 4]>,
+    // Start, end (both inclusive)
+    map: SmallVec<[(u32, u32); 2]>,
     domain: usize,
     _data: PhantomData<I>,
 }
diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs
index 234dc5133f8..63a729b13dc 100644
--- a/compiler/rustc_infer/src/infer/mod.rs
+++ b/compiler/rustc_infer/src/infer/mod.rs
@@ -458,8 +458,8 @@ pub enum RegionVariableOrigin {
     PatternRegion(Span),
 
     /// Regions created by `&` operator.
-    ///
-    AddrOfRegion(Span),
+    BorrowRegion(Span),
+
     /// Regions created as part of an autoref of a method receiver.
     Autoref(Span),
 
@@ -1741,7 +1741,7 @@ impl RegionVariableOrigin {
         match *self {
             MiscVariable(a)
             | PatternRegion(a)
-            | AddrOfRegion(a)
+            | BorrowRegion(a)
             | Autoref(a)
             | Coercion(a)
             | RegionParameterDefinition(a, ..)
diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs
index 42fed98df01..50422df8ee6 100644
--- a/compiler/rustc_interface/src/tests.rs
+++ b/compiler/rustc_interface/src/tests.rs
@@ -808,7 +808,7 @@ fn test_unstable_options_tracking_hash() {
     tracked!(mir_opt_level, Some(4));
     tracked!(move_size_limit, Some(4096));
     tracked!(mutable_noalias, false);
-    tracked!(next_solver, Some(NextSolverConfig { coherence: true, globally: false }));
+    tracked!(next_solver, NextSolverConfig { coherence: true, globally: true });
     tracked!(no_generate_arange_section, true);
     tracked!(no_jump_tables, true);
     tracked!(no_link, true);
diff --git a/compiler/rustc_lexer/src/lib.rs b/compiler/rustc_lexer/src/lib.rs
index 31fdd2d7cec..60aab668cba 100644
--- a/compiler/rustc_lexer/src/lib.rs
+++ b/compiler/rustc_lexer/src/lib.rs
@@ -91,6 +91,15 @@ pub enum TokenKind {
     /// tokens.
     UnknownPrefix,
 
+    /// An unknown prefix in a lifetime, like `'foo#`.
+    ///
+    /// Note that like above, only the `'` and prefix are included in the token
+    /// and not the separator.
+    UnknownPrefixLifetime,
+
+    /// `'r#lt`, which in edition < 2021 is split into several tokens: `'r # lt`.
+    RawLifetime,
+
     /// Similar to the above, but *always* an error on every edition. This is used
     /// for emoji identifier recovery, as those are not meant to be ever accepted.
     InvalidPrefix,
@@ -677,9 +686,17 @@ impl Cursor<'_> {
             return Literal { kind, suffix_start };
         }
 
+        if self.first() == 'r' && self.second() == '#' && is_id_start(self.third()) {
+            // Eat "r" and `#`, and identifier start characters.
+            self.bump();
+            self.bump();
+            self.bump();
+            self.eat_while(is_id_continue);
+            return RawLifetime;
+        }
+
         // Either a lifetime or a character literal with
         // length greater than 1.
-
         let starts_with_number = self.first().is_ascii_digit();
 
         // Skip the literal contents.
@@ -688,15 +705,17 @@ impl Cursor<'_> {
         self.bump();
         self.eat_while(is_id_continue);
 
-        // Check if after skipping literal contents we've met a closing
-        // single quote (which means that user attempted to create a
-        // string with single quotes).
-        if self.first() == '\'' {
-            self.bump();
-            let kind = Char { terminated: true };
-            Literal { kind, suffix_start: self.pos_within_token() }
-        } else {
-            Lifetime { starts_with_number }
+        match self.first() {
+            // Check if after skipping literal contents we've met a closing
+            // single quote (which means that user attempted to create a
+            // string with single quotes).
+            '\'' => {
+                self.bump();
+                let kind = Char { terminated: true };
+                Literal { kind, suffix_start: self.pos_within_token() }
+            }
+            '#' if !starts_with_number => UnknownPrefixLifetime,
+            _ => Lifetime { starts_with_number },
         }
     }
 
diff --git a/compiler/rustc_lint/messages.ftl b/compiler/rustc_lint/messages.ftl
index 35334595833..7d4dee45c26 100644
--- a/compiler/rustc_lint/messages.ftl
+++ b/compiler/rustc_lint/messages.ftl
@@ -255,6 +255,7 @@ lint_duplicate_matcher_binding = duplicate matcher binding
 lint_elided_named_lifetime = elided lifetime has a name
     .label_elided = this elided lifetime gets resolved as `{$name}`
     .label_named = lifetime `{$name}` declared here
+    .suggestion = consider specifying it explicitly
 
 lint_enum_intrinsics_mem_discriminant =
     the return value of `mem::discriminant` is unspecified when called with a non-enum type
@@ -699,11 +700,18 @@ lint_ptr_null_checks_ref = references are not nullable, so checking them for nul
 lint_query_instability = using `{$query}` can result in unstable query results
     .note = if you believe this case to be fine, allow this lint and add a comment explaining your rationale
 
+lint_query_untracked = `{$method}` accesses information that is not tracked by the query system
+    .note = if you believe this case to be fine, allow this lint and add a comment explaining your rationale
+
 lint_range_endpoint_out_of_range = range endpoint is out of range for `{$ty}`
 
 lint_range_use_inclusive_range = use an inclusive range instead
 
 
+lint_raw_prefix = prefix `'r` is reserved
+    .label = reserved prefix
+    .suggestion = insert whitespace here to avoid this being parsed as a prefix in Rust 2021
+
 lint_reason_must_be_string_literal = reason must be a string literal
 
 lint_reason_must_come_last = reason in lint attribute must come last
diff --git a/compiler/rustc_lint/src/builtin.rs b/compiler/rustc_lint/src/builtin.rs
index d8482567bbe..824a1868c55 100644
--- a/compiler/rustc_lint/src/builtin.rs
+++ b/compiler/rustc_lint/src/builtin.rs
@@ -1851,9 +1851,16 @@ impl KeywordIdents {
                 TokenTree::Token(token, _) => {
                     if let Some((ident, token::IdentIsRaw::No)) = token.ident() {
                         if !prev_dollar {
-                            self.check_ident_token(cx, UnderMacro(true), ident);
+                            self.check_ident_token(cx, UnderMacro(true), ident, "");
                         }
-                    } else if *token == TokenKind::Dollar {
+                    } else if let Some((ident, token::IdentIsRaw::No)) = token.lifetime() {
+                        self.check_ident_token(
+                            cx,
+                            UnderMacro(true),
+                            ident.without_first_quote(),
+                            "'",
+                        );
+                    } else if token.kind == TokenKind::Dollar {
                         prev_dollar = true;
                         continue;
                     }
@@ -1869,6 +1876,7 @@ impl KeywordIdents {
         cx: &EarlyContext<'_>,
         UnderMacro(under_macro): UnderMacro,
         ident: Ident,
+        prefix: &'static str,
     ) {
         let (lint, edition) = match ident.name {
             kw::Async | kw::Await | kw::Try => (KEYWORD_IDENTS_2018, Edition::Edition2018),
@@ -1902,7 +1910,7 @@ impl KeywordIdents {
         cx.emit_span_lint(
             lint,
             ident.span,
-            BuiltinKeywordIdents { kw: ident, next: edition, suggestion: ident.span },
+            BuiltinKeywordIdents { kw: ident, next: edition, suggestion: ident.span, prefix },
         );
     }
 }
@@ -1915,7 +1923,11 @@ impl EarlyLintPass for KeywordIdents {
         self.check_tokens(cx, &mac.args.tokens);
     }
     fn check_ident(&mut self, cx: &EarlyContext<'_>, ident: Ident) {
-        self.check_ident_token(cx, UnderMacro(false), ident);
+        if ident.name.as_str().starts_with('\'') {
+            self.check_ident_token(cx, UnderMacro(false), ident.without_first_quote(), "'");
+        } else {
+            self.check_ident_token(cx, UnderMacro(false), ident, "");
+        }
     }
 }
 
diff --git a/compiler/rustc_lint/src/context/diagnostics.rs b/compiler/rustc_lint/src/context/diagnostics.rs
index fd43afa1743..fd13c418c09 100644
--- a/compiler/rustc_lint/src/context/diagnostics.rs
+++ b/compiler/rustc_lint/src/context/diagnostics.rs
@@ -8,13 +8,13 @@ use rustc_errors::{
     elided_lifetime_in_path_suggestion, Applicability, Diag, DiagArgValue, LintDiagnostic,
 };
 use rustc_middle::middle::stability;
-use rustc_session::lint::BuiltinLintDiag;
+use rustc_session::lint::{BuiltinLintDiag, ElidedLifetimeResolution};
 use rustc_session::Session;
 use rustc_span::symbol::kw;
 use rustc_span::BytePos;
 use tracing::debug;
 
-use crate::lints;
+use crate::lints::{self, ElidedNamedLifetime};
 
 mod check_cfg;
 
@@ -172,6 +172,10 @@ pub(super) fn decorate_lint(sess: &Session, diagnostic: BuiltinLintDiag, diag: &
             }
             .decorate_lint(diag);
         }
+        BuiltinLintDiag::RawPrefix(label_span) => {
+            lints::RawPrefix { label: label_span, suggestion: label_span.shrink_to_hi() }
+                .decorate_lint(diag);
+        }
         BuiltinLintDiag::UnusedBuiltinAttribute { attr_name, macro_name, invoc_span } => {
             lints::UnusedBuiltinAttribute { invoc_span, attr_name, macro_name }.decorate_lint(diag);
         }
@@ -442,15 +446,14 @@ pub(super) fn decorate_lint(sess: &Session, diagnostic: BuiltinLintDiag, diag: &
         BuiltinLintDiag::UnexpectedBuiltinCfg { cfg, cfg_name, controlled_by } => {
             lints::UnexpectedBuiltinCfg { cfg, cfg_name, controlled_by }.decorate_lint(diag)
         }
-        BuiltinLintDiag::ElidedIsStatic { elided } => {
-            lints::ElidedNamedLifetime { elided, name: kw::StaticLifetime, named_declaration: None }
-                .decorate_lint(diag)
-        }
-        BuiltinLintDiag::ElidedIsParam { elided, param: (param_name, param_span) } => {
-            lints::ElidedNamedLifetime {
-                elided,
-                name: param_name,
-                named_declaration: Some(param_span),
+        BuiltinLintDiag::ElidedNamedLifetimes { elided: (span, kind), resolution } => {
+            match resolution {
+                ElidedLifetimeResolution::Static => {
+                    ElidedNamedLifetime { span, kind, name: kw::StaticLifetime, declaration: None }
+                }
+                ElidedLifetimeResolution::Param(name, declaration) => {
+                    ElidedNamedLifetime { span, kind, name, declaration: Some(declaration) }
+                }
             }
             .decorate_lint(diag)
         }
diff --git a/compiler/rustc_lint/src/internal.rs b/compiler/rustc_lint/src/internal.rs
index 2e8116b8ba8..9d637c1eb7f 100644
--- a/compiler/rustc_lint/src/internal.rs
+++ b/compiler/rustc_lint/src/internal.rs
@@ -17,8 +17,8 @@ use tracing::debug;
 
 use crate::lints::{
     BadOptAccessDiag, DefaultHashTypesDiag, DiagOutOfImpl, LintPassByHand, NonExistentDocKeyword,
-    NonGlobImportTypeIrInherent, QueryInstability, SpanUseEqCtxtDiag, TyQualified, TykindDiag,
-    TykindKind, TypeIrInherentUsage, UntranslatableDiag,
+    NonGlobImportTypeIrInherent, QueryInstability, QueryUntracked, SpanUseEqCtxtDiag, TyQualified,
+    TykindDiag, TykindKind, TypeIrInherentUsage, UntranslatableDiag,
 };
 use crate::{EarlyContext, EarlyLintPass, LateContext, LateLintPass, LintContext};
 
@@ -88,7 +88,18 @@ declare_tool_lint! {
     report_in_external_macro: true
 }
 
-declare_lint_pass!(QueryStability => [POTENTIAL_QUERY_INSTABILITY]);
+declare_tool_lint! {
+    /// The `untracked_query_information` lint detects use of methods which leak information not
+    /// tracked by the query system, such as whether a `Steal<T>` value has already been stolen. In
+    /// order not to break incremental compilation, such methods must be used very carefully or not
+    /// at all.
+    pub rustc::UNTRACKED_QUERY_INFORMATION,
+    Allow,
+    "require explicit opt-in when accessing information not tracked by the query system",
+    report_in_external_macro: true
+}
+
+declare_lint_pass!(QueryStability => [POTENTIAL_QUERY_INSTABILITY, UNTRACKED_QUERY_INFORMATION]);
 
 impl LateLintPass<'_> for QueryStability {
     fn check_expr(&mut self, cx: &LateContext<'_>, expr: &Expr<'_>) {
@@ -102,6 +113,13 @@ impl LateLintPass<'_> for QueryStability {
                     QueryInstability { query: cx.tcx.item_name(def_id) },
                 );
             }
+            if cx.tcx.has_attr(def_id, sym::rustc_lint_untracked_query_information) {
+                cx.emit_span_lint(
+                    UNTRACKED_QUERY_INFORMATION,
+                    span,
+                    QueryUntracked { method: cx.tcx.item_name(def_id) },
+                );
+            }
         }
     }
 }
diff --git a/compiler/rustc_lint/src/lib.rs b/compiler/rustc_lint/src/lib.rs
index bb7de4739fb..4139ac4e3e9 100644
--- a/compiler/rustc_lint/src/lib.rs
+++ b/compiler/rustc_lint/src/lib.rs
@@ -576,6 +576,10 @@ fn register_builtins(store: &mut LintStore) {
          <https://github.com/rust-lang/rust/issues/107457> for more information",
     );
     store.register_removed("writes_through_immutable_pointer", "converted into hard error");
+    store.register_removed(
+        "const_eval_mutable_ptr_in_final_value",
+        "partially allowed now, otherwise turned into a hard error",
+    );
 }
 
 fn register_internals(store: &mut LintStore) {
@@ -610,6 +614,7 @@ fn register_internals(store: &mut LintStore) {
         vec![
             LintId::of(DEFAULT_HASH_TYPES),
             LintId::of(POTENTIAL_QUERY_INSTABILITY),
+            LintId::of(UNTRACKED_QUERY_INFORMATION),
             LintId::of(USAGE_OF_TY_TYKIND),
             LintId::of(PASS_BY_VALUE),
             LintId::of(LINT_PASS_IMPL_WITHOUT_MACRO),
diff --git a/compiler/rustc_lint/src/lints.rs b/compiler/rustc_lint/src/lints.rs
index 7ca282b7c85..e49b102cb39 100644
--- a/compiler/rustc_lint/src/lints.rs
+++ b/compiler/rustc_lint/src/lints.rs
@@ -9,7 +9,7 @@ use rustc_errors::{
 };
 use rustc_hir::def::Namespace;
 use rustc_hir::def_id::DefId;
-use rustc_hir::{self as hir};
+use rustc_hir::{self as hir, MissingLifetimeKind};
 use rustc_macros::{LintDiagnostic, Subdiagnostic};
 use rustc_middle::ty::inhabitedness::InhabitedPredicate;
 use rustc_middle::ty::{Clause, PolyExistentialTraitRef, Ty, TyCtxt};
@@ -363,8 +363,9 @@ pub(crate) enum BuiltinEllipsisInclusiveRangePatternsLint {
 pub(crate) struct BuiltinKeywordIdents {
     pub kw: Ident,
     pub next: Edition,
-    #[suggestion(code = "r#{kw}", applicability = "machine-applicable")]
+    #[suggestion(code = "{prefix}r#{kw}", applicability = "machine-applicable")]
     pub suggestion: Span,
+    pub prefix: &'static str,
 }
 
 #[derive(LintDiagnostic)]
@@ -895,6 +896,13 @@ pub(crate) struct QueryInstability {
 }
 
 #[derive(LintDiagnostic)]
+#[diag(lint_query_untracked)]
+#[note]
+pub(crate) struct QueryUntracked {
+    pub method: Symbol,
+}
+
+#[derive(LintDiagnostic)]
 #[diag(lint_span_use_eq_ctxt)]
 pub(crate) struct SpanUseEqCtxtDiag;
 
@@ -2616,14 +2624,56 @@ pub(crate) struct ElidedLifetimesInPaths {
     pub subdiag: ElidedLifetimeInPathSubdiag,
 }
 
-#[derive(LintDiagnostic)]
-#[diag(lint_elided_named_lifetime)]
 pub(crate) struct ElidedNamedLifetime {
-    #[label(lint_label_elided)]
-    pub elided: Span,
+    pub span: Span,
+    pub kind: MissingLifetimeKind,
     pub name: Symbol,
-    #[label(lint_label_named)]
-    pub named_declaration: Option<Span>,
+    pub declaration: Option<Span>,
+}
+
+impl<G: EmissionGuarantee> LintDiagnostic<'_, G> for ElidedNamedLifetime {
+    fn decorate_lint(self, diag: &mut rustc_errors::Diag<'_, G>) {
+        let Self { span, kind, name, declaration } = self;
+        diag.primary_message(fluent::lint_elided_named_lifetime);
+        diag.arg("name", name);
+        diag.span_label(span, fluent::lint_label_elided);
+        if let Some(declaration) = declaration {
+            diag.span_label(declaration, fluent::lint_label_named);
+        }
+        // FIXME(GrigorenkoPV): this `if` and `return` should be removed,
+        //  but currently this lint's suggestions can conflict with those of `clippy::needless_lifetimes`:
+        //  https://github.com/rust-lang/rust/pull/129840#issuecomment-2323349119
+        // HACK: `'static` suggestions will never sonflict, emit only those for now.
+        if name != rustc_span::symbol::kw::StaticLifetime {
+            return;
+        }
+        match kind {
+            MissingLifetimeKind::Underscore => diag.span_suggestion_verbose(
+                span,
+                fluent::lint_suggestion,
+                format!("{name}"),
+                Applicability::MachineApplicable,
+            ),
+            MissingLifetimeKind::Ampersand => diag.span_suggestion_verbose(
+                span.shrink_to_hi(),
+                fluent::lint_suggestion,
+                format!("{name} "),
+                Applicability::MachineApplicable,
+            ),
+            MissingLifetimeKind::Comma => diag.span_suggestion_verbose(
+                span.shrink_to_hi(),
+                fluent::lint_suggestion,
+                format!("{name}, "),
+                Applicability::MachineApplicable,
+            ),
+            MissingLifetimeKind::Brackets => diag.span_suggestion_verbose(
+                span.shrink_to_hi(),
+                fluent::lint_suggestion,
+                format!("<{name}>"),
+                Applicability::MachineApplicable,
+            ),
+        };
+    }
 }
 
 #[derive(LintDiagnostic)]
@@ -2766,6 +2816,15 @@ pub(crate) struct ReservedPrefix {
 }
 
 #[derive(LintDiagnostic)]
+#[diag(lint_raw_prefix)]
+pub(crate) struct RawPrefix {
+    #[label]
+    pub label: Span,
+    #[suggestion(code = " ", applicability = "machine-applicable")]
+    pub suggestion: Span,
+}
+
+#[derive(LintDiagnostic)]
 #[diag(lint_unused_builtin_attribute)]
 pub(crate) struct UnusedBuiltinAttribute {
     #[note]
diff --git a/compiler/rustc_lint/src/shadowed_into_iter.rs b/compiler/rustc_lint/src/shadowed_into_iter.rs
index bb9c7d85c2e..bb122509d0a 100644
--- a/compiler/rustc_lint/src/shadowed_into_iter.rs
+++ b/compiler/rustc_lint/src/shadowed_into_iter.rs
@@ -94,12 +94,9 @@ impl<'tcx> LateLintPass<'tcx> for ShadowedIntoIter {
         fn is_ref_to_array(ty: Ty<'_>) -> bool {
             if let ty::Ref(_, pointee_ty, _) = *ty.kind() { pointee_ty.is_array() } else { false }
         }
-        fn is_boxed_slice(ty: Ty<'_>) -> bool {
-            ty.is_box() && ty.boxed_ty().is_slice()
-        }
         fn is_ref_to_boxed_slice(ty: Ty<'_>) -> bool {
             if let ty::Ref(_, pointee_ty, _) = *ty.kind() {
-                is_boxed_slice(pointee_ty)
+                pointee_ty.boxed_ty().is_some_and(Ty::is_slice)
             } else {
                 false
             }
@@ -119,7 +116,7 @@ impl<'tcx> LateLintPass<'tcx> for ShadowedIntoIter {
                     .iter()
                     .copied()
                     .take_while(|ty| !is_ref_to_boxed_slice(*ty))
-                    .position(|ty| is_boxed_slice(ty))
+                    .position(|ty| ty.boxed_ty().is_some_and(Ty::is_slice))
             {
                 (BOXED_SLICE_INTO_ITER, "Box<[T]>", "2024", idx == 0)
             } else {
diff --git a/compiler/rustc_lint/src/types.rs b/compiler/rustc_lint/src/types.rs
index f2f7c0eaa4d..b5e501b92f0 100644
--- a/compiler/rustc_lint/src/types.rs
+++ b/compiler/rustc_lint/src/types.rs
@@ -1304,8 +1304,10 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
 
         match *ty.kind() {
             ty::Adt(def, args) => {
-                if def.is_box() && matches!(self.mode, CItemKind::Definition) {
-                    if ty.boxed_ty().is_sized(tcx, self.cx.param_env) {
+                if let Some(boxed) = ty.boxed_ty()
+                    && matches!(self.mode, CItemKind::Definition)
+                {
+                    if boxed.is_sized(tcx, self.cx.param_env) {
                         return FfiSafe;
                     } else {
                         return FfiUnsafe {
diff --git a/compiler/rustc_lint/src/unused.rs b/compiler/rustc_lint/src/unused.rs
index 761d30bac71..b7f7b782c77 100644
--- a/compiler/rustc_lint/src/unused.rs
+++ b/compiler/rustc_lint/src/unused.rs
@@ -283,9 +283,8 @@ impl<'tcx> LateLintPass<'tcx> for UnusedResults {
             }
 
             match *ty.kind() {
-                ty::Adt(..) if ty.is_box() => {
-                    let boxed_ty = ty.boxed_ty();
-                    is_ty_must_use(cx, boxed_ty, expr, span)
+                ty::Adt(..) if let Some(boxed) = ty.boxed_ty() => {
+                    is_ty_must_use(cx, boxed, expr, span)
                         .map(|inner| MustUsePath::Boxed(Box::new(inner)))
                 }
                 ty::Adt(def, args) if cx.tcx.is_lang_item(def.did(), LangItem::Pin) => {
diff --git a/compiler/rustc_lint_defs/src/builtin.rs b/compiler/rustc_lint_defs/src/builtin.rs
index 25d33126754..f42f35c594b 100644
--- a/compiler/rustc_lint_defs/src/builtin.rs
+++ b/compiler/rustc_lint_defs/src/builtin.rs
@@ -29,7 +29,6 @@ declare_lint_pass! {
         CENUM_IMPL_DROP_CAST,
         COHERENCE_LEAK_CHECK,
         CONFLICTING_REPR_HINTS,
-        CONST_EVAL_MUTABLE_PTR_IN_FINAL_VALUE,
         CONST_EVALUATABLE_UNCHECKED,
         CONST_ITEM_MUTATION,
         DEAD_CODE,
@@ -2805,51 +2804,6 @@ declare_lint! {
 }
 
 declare_lint! {
-    /// The `const_eval_mutable_ptr_in_final_value` lint detects if a mutable pointer
-    /// has leaked into the final value of a const expression.
-    ///
-    /// ### Example
-    ///
-    /// ```rust
-    /// pub enum JsValue {
-    ///     Undefined,
-    ///     Object(std::cell::Cell<bool>),
-    /// }
-    ///
-    /// impl ::std::ops::Drop for JsValue {
-    ///     fn drop(&mut self) {}
-    /// }
-    ///
-    /// const UNDEFINED: &JsValue = &JsValue::Undefined;
-    ///
-    /// fn main() {
-    /// }
-    /// ```
-    ///
-    /// {{produces}}
-    ///
-    /// ### Explanation
-    ///
-    /// In the 1.77 release, the const evaluation machinery adopted some
-    /// stricter rules to reject expressions with values that could
-    /// end up holding mutable references to state stored in static memory
-    /// (which is inherently immutable).
-    ///
-    /// This is a [future-incompatible] lint to ease the transition to an error.
-    /// See [issue #122153] for more details.
-    ///
-    /// [issue #122153]: https://github.com/rust-lang/rust/issues/122153
-    /// [future-incompatible]: ../index.md#future-incompatible-lints
-    pub CONST_EVAL_MUTABLE_PTR_IN_FINAL_VALUE,
-    Warn,
-    "detects a mutable pointer that has leaked into final value of a const expression",
-    @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorReportInDeps,
-        reference: "issue #122153 <https://github.com/rust-lang/rust/issues/122153>",
-    };
-}
-
-declare_lint! {
     /// The `const_evaluatable_unchecked` lint detects a generic constant used
     /// in a type.
     ///
diff --git a/compiler/rustc_lint_defs/src/lib.rs b/compiler/rustc_lint_defs/src/lib.rs
index 6ee33041623..5c4b7e5664d 100644
--- a/compiler/rustc_lint_defs/src/lib.rs
+++ b/compiler/rustc_lint_defs/src/lib.rs
@@ -10,7 +10,7 @@ use rustc_data_structures::stable_hasher::{
 };
 use rustc_error_messages::{DiagMessage, MultiSpan};
 use rustc_hir::def::Namespace;
-use rustc_hir::{HashStableContext, HirId};
+use rustc_hir::{HashStableContext, HirId, MissingLifetimeKind};
 use rustc_macros::{Decodable, Encodable, HashStable_Generic};
 use rustc_span::edition::Edition;
 use rustc_span::symbol::{Ident, MacroRulesNormalizedIdent};
@@ -556,6 +556,12 @@ pub enum DeprecatedSinceKind {
     InVersion(String),
 }
 
+#[derive(Debug)]
+pub enum ElidedLifetimeResolution {
+    Static,
+    Param(Symbol, Span),
+}
+
 // This could be a closure, but then implementing derive trait
 // becomes hacky (and it gets allocated).
 #[derive(Debug)]
@@ -568,12 +574,9 @@ pub enum BuiltinLintDiag {
     },
     MacroExpandedMacroExportsAccessedByAbsolutePaths(Span),
     ElidedLifetimesInPaths(usize, Span, bool, Span),
-    ElidedIsStatic {
-        elided: Span,
-    },
-    ElidedIsParam {
-        elided: Span,
-        param: (Symbol, Span),
+    ElidedNamedLifetimes {
+        elided: (Span, MissingLifetimeKind),
+        resolution: ElidedLifetimeResolution,
     },
     UnknownCrateTypes {
         span: Span,
@@ -609,6 +612,8 @@ pub enum BuiltinLintDiag {
     LegacyDeriveHelpers(Span),
     OrPatternsBackCompat(Span, String),
     ReservedPrefix(Span, String),
+    /// `'r#` in edition < 2021.
+    RawPrefix(Span),
     TrailingMacro(bool, Ident),
     BreakWithLabelAndLoop(Span),
     UnicodeTextFlow(Span, String),
diff --git a/compiler/rustc_llvm/src/lib.rs b/compiler/rustc_llvm/src/lib.rs
index 3532d50c64e..055d2bd5bc9 100644
--- a/compiler/rustc_llvm/src/lib.rs
+++ b/compiler/rustc_llvm/src/lib.rs
@@ -1,6 +1,5 @@
 // tidy-alphabetical-start
 #![allow(internal_features)]
-#![cfg_attr(bootstrap, feature(unsafe_attributes, unsafe_extern_blocks))]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(rustdoc_internals)]
diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs
index c55583b39a6..b617d5236b9 100644
--- a/compiler/rustc_metadata/src/rmeta/encoder.rs
+++ b/compiler/rustc_metadata/src/rmeta/encoder.rs
@@ -9,6 +9,7 @@ use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_data_structures::memmap::{Mmap, MmapMut};
 use rustc_data_structures::sync::{join, par_for_each_in, Lrc};
 use rustc_data_structures::temp_dir::MaybeTempDir;
+use rustc_feature::Features;
 use rustc_hir as hir;
 use rustc_hir::def_id::{LocalDefId, LocalDefIdSet, CRATE_DEF_ID, CRATE_DEF_INDEX, LOCAL_CRATE};
 use rustc_hir::definitions::DefPathData;
@@ -797,9 +798,10 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
     }
 }
 
-struct AnalyzeAttrState {
+struct AnalyzeAttrState<'a> {
     is_exported: bool,
     is_doc_hidden: bool,
+    features: &'a Features,
 }
 
 /// Returns whether an attribute needs to be recorded in metadata, that is, if it's usable and
@@ -812,7 +814,7 @@ struct AnalyzeAttrState {
 /// visibility: this is a piece of data that can be computed once per defid, and not once per
 /// attribute. Some attributes would only be usable downstream if they are public.
 #[inline]
-fn analyze_attr(attr: &Attribute, state: &mut AnalyzeAttrState) -> bool {
+fn analyze_attr(attr: &Attribute, state: &mut AnalyzeAttrState<'_>) -> bool {
     let mut should_encode = false;
     if !rustc_feature::encode_cross_crate(attr.name_or_empty()) {
         // Attributes not marked encode-cross-crate don't need to be encoded for downstream crates.
@@ -837,6 +839,9 @@ fn analyze_attr(attr: &Attribute, state: &mut AnalyzeAttrState) -> bool {
                 }
             }
         }
+    } else if attr.path().starts_with(&[sym::diagnostic]) && attr.path().len() == 2 {
+        should_encode =
+            rustc_feature::is_stable_diagnostic_attribute(attr.path()[1], state.features);
     } else {
         should_encode = true;
     }
@@ -1343,6 +1348,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         let mut state = AnalyzeAttrState {
             is_exported: tcx.effective_visibilities(()).is_exported(def_id),
             is_doc_hidden: false,
+            features: &tcx.features(),
         };
         let attr_iter = tcx
             .hir()
@@ -2033,7 +2039,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 let simplified_self_ty = fast_reject::simplify_type(
                     self.tcx,
                     trait_ref.self_ty(),
-                    TreatParams::AsCandidateKey,
+                    TreatParams::InstantiateWithInfer,
                 );
                 trait_impls
                     .entry(trait_ref.def_id)
diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs
index 2a85f85a9f4..fd6e2ad79b1 100644
--- a/compiler/rustc_middle/src/lib.rs
+++ b/compiler/rustc_middle/src/lib.rs
@@ -28,7 +28,6 @@
 #![allow(rustc::diagnostic_outside_of_impl)]
 #![allow(rustc::potential_query_instability)]
 #![allow(rustc::untranslatable_diagnostic)]
-#![cfg_attr(bootstrap, feature(min_exhaustive_patterns, unsafe_extern_blocks))]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(allocator_api)]
diff --git a/compiler/rustc_middle/src/mir/interpret/allocation.rs b/compiler/rustc_middle/src/mir/interpret/allocation.rs
index 5fb8af576ae..cd56d0edc05 100644
--- a/compiler/rustc_middle/src/mir/interpret/allocation.rs
+++ b/compiler/rustc_middle/src/mir/interpret/allocation.rs
@@ -644,6 +644,12 @@ impl<Prov: Provenance, Extra, Bytes: AllocBytes> Allocation<Prov, Extra, Bytes>
         return Ok(());
     }
 
+    /// Remove all provenance in the given memory range.
+    pub fn clear_provenance(&mut self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult {
+        self.provenance.clear(range, cx)?;
+        return Ok(());
+    }
+
     /// Applies a previously prepared provenance copy.
     /// The affected range, as defined in the parameters to `provenance().prepare_copy` is expected
     /// to be clear of provenance.
diff --git a/compiler/rustc_middle/src/mir/interpret/pointer.rs b/compiler/rustc_middle/src/mir/interpret/pointer.rs
index 6cfd07d699c..1700b0f02ec 100644
--- a/compiler/rustc_middle/src/mir/interpret/pointer.rs
+++ b/compiler/rustc_middle/src/mir/interpret/pointer.rs
@@ -80,14 +80,23 @@ pub trait Provenance: Copy + fmt::Debug + 'static {
 }
 
 /// The type of provenance in the compile-time interpreter.
-/// This is a packed representation of an `AllocId` and an `immutable: bool`.
+/// This is a packed representation of:
+/// - an `AllocId` (non-zero)
+/// - an `immutable: bool`
+/// - a `shared_ref: bool`
+///
+/// with the extra invariant that if `immutable` is `true`, then so
+/// is `shared_ref`.
 #[derive(Copy, Clone, Eq, Hash, Ord, PartialEq, PartialOrd)]
 pub struct CtfeProvenance(NonZero<u64>);
 
 impl From<AllocId> for CtfeProvenance {
     fn from(value: AllocId) -> Self {
         let prov = CtfeProvenance(value.0);
-        assert!(!prov.immutable(), "`AllocId` with the highest bit set cannot be used in CTFE");
+        assert!(
+            prov.alloc_id() == value,
+            "`AllocId` with the highest bits set cannot be used in CTFE"
+        );
         prov
     }
 }
@@ -103,12 +112,14 @@ impl fmt::Debug for CtfeProvenance {
 }
 
 const IMMUTABLE_MASK: u64 = 1 << 63; // the highest bit
+const SHARED_REF_MASK: u64 = 1 << 62;
+const ALLOC_ID_MASK: u64 = u64::MAX & !IMMUTABLE_MASK & !SHARED_REF_MASK;
 
 impl CtfeProvenance {
     /// Returns the `AllocId` of this provenance.
     #[inline(always)]
     pub fn alloc_id(self) -> AllocId {
-        AllocId(NonZero::new(self.0.get() & !IMMUTABLE_MASK).unwrap())
+        AllocId(NonZero::new(self.0.get() & ALLOC_ID_MASK).unwrap())
     }
 
     /// Returns whether this provenance is immutable.
@@ -117,10 +128,38 @@ impl CtfeProvenance {
         self.0.get() & IMMUTABLE_MASK != 0
     }
 
+    /// Returns whether this provenance is derived from a shared reference.
+    #[inline]
+    pub fn shared_ref(self) -> bool {
+        self.0.get() & SHARED_REF_MASK != 0
+    }
+
+    pub fn into_parts(self) -> (AllocId, bool, bool) {
+        (self.alloc_id(), self.immutable(), self.shared_ref())
+    }
+
+    pub fn from_parts((alloc_id, immutable, shared_ref): (AllocId, bool, bool)) -> Self {
+        let prov = CtfeProvenance::from(alloc_id);
+        if immutable {
+            // This sets both flags, so we don't even have to check `shared_ref`.
+            prov.as_immutable()
+        } else if shared_ref {
+            prov.as_shared_ref()
+        } else {
+            prov
+        }
+    }
+
     /// Returns an immutable version of this provenance.
     #[inline]
     pub fn as_immutable(self) -> Self {
-        CtfeProvenance(self.0 | IMMUTABLE_MASK)
+        CtfeProvenance(self.0 | IMMUTABLE_MASK | SHARED_REF_MASK)
+    }
+
+    /// Returns a "shared reference" (but not necessarily immutable!) version of this provenance.
+    #[inline]
+    pub fn as_shared_ref(self) -> Self {
+        CtfeProvenance(self.0 | SHARED_REF_MASK)
     }
 }
 
diff --git a/compiler/rustc_middle/src/mir/interpret/value.rs b/compiler/rustc_middle/src/mir/interpret/value.rs
index 84c17b39a62..989f03d3d13 100644
--- a/compiler/rustc_middle/src/mir/interpret/value.rs
+++ b/compiler/rustc_middle/src/mir/interpret/value.rs
@@ -307,6 +307,13 @@ impl<'tcx, Prov: Provenance> Scalar<Prov> {
         }
     }
 
+    pub fn clear_provenance(&mut self) -> InterpResult<'tcx> {
+        if matches!(self, Scalar::Ptr(..)) {
+            *self = self.to_scalar_int()?.into();
+        }
+        Ok(())
+    }
+
     #[inline(always)]
     pub fn to_scalar_int(self) -> InterpResult<'tcx, ScalarInt> {
         self.try_to_scalar_int().map_err(|_| err_unsup!(ReadPointerAsInt(None)).into())
diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs
index 748ca047754..22a4b688c51 100644
--- a/compiler/rustc_middle/src/mir/syntax.rs
+++ b/compiler/rustc_middle/src/mir/syntax.rs
@@ -1307,6 +1307,9 @@ pub enum Rvalue<'tcx> {
     /// If the type of the place is an array, this is the array length. For slices (`[T]`, not
     /// `&[T]`) this accesses the place's metadata to determine the length. This rvalue is
     /// ill-formed for places of other types.
+    ///
+    /// This cannot be a `UnOp(PtrMetadata, _)` because that expects a value, and we only
+    /// have a place, and `UnOp(PtrMetadata, RawPtr(place))` is not a thing.
     Len(Place<'tcx>),
 
     /// Performs essentially all of the casts that can be performed via `as`.
diff --git a/compiler/rustc_middle/src/ty/closure.rs b/compiler/rustc_middle/src/ty/closure.rs
index 8eb3c015679..809801c33e1 100644
--- a/compiler/rustc_middle/src/ty/closure.rs
+++ b/compiler/rustc_middle/src/ty/closure.rs
@@ -88,9 +88,6 @@ pub struct CapturedPlace<'tcx> {
 
     /// Represents if `place` can be mutated or not.
     pub mutability: hir::Mutability,
-
-    /// Region of the resulting reference if the upvar is captured by ref.
-    pub region: Option<ty::Region<'tcx>>,
 }
 
 impl<'tcx> CapturedPlace<'tcx> {
diff --git a/compiler/rustc_middle/src/ty/codec.rs b/compiler/rustc_middle/src/ty/codec.rs
index 46203ee150f..7e533bc4291 100644
--- a/compiler/rustc_middle/src/ty/codec.rs
+++ b/compiler/rustc_middle/src/ty/codec.rs
@@ -165,8 +165,7 @@ impl<'tcx, E: TyEncoder<I = TyCtxt<'tcx>>> Encodable<E> for AllocId {
 
 impl<'tcx, E: TyEncoder<I = TyCtxt<'tcx>>> Encodable<E> for CtfeProvenance {
     fn encode(&self, e: &mut E) {
-        self.alloc_id().encode(e);
-        self.immutable().encode(e);
+        self.into_parts().encode(e);
     }
 }
 
@@ -295,10 +294,8 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for AllocId {
 
 impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for CtfeProvenance {
     fn decode(decoder: &mut D) -> Self {
-        let alloc_id: AllocId = Decodable::decode(decoder);
-        let prov = CtfeProvenance::from(alloc_id);
-        let immutable: bool = Decodable::decode(decoder);
-        if immutable { prov.as_immutable() } else { prov }
+        let parts = Decodable::decode(decoder);
+        CtfeProvenance::from_parts(parts)
     }
 }
 
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index 20828067c46..5334e767766 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -426,7 +426,7 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
                 let simp = ty::fast_reject::simplify_type(
                     tcx,
                     self_ty,
-                    ty::fast_reject::TreatParams::ForLookup,
+                    ty::fast_reject::TreatParams::AsRigid,
                 )
                 .unwrap();
                 consider_impls_for_simplified_type(simp);
@@ -3128,11 +3128,11 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     pub fn next_trait_solver_globally(self) -> bool {
-        self.sess.opts.unstable_opts.next_solver.map_or(false, |c| c.globally)
+        self.sess.opts.unstable_opts.next_solver.globally
     }
 
     pub fn next_trait_solver_in_coherence(self) -> bool {
-        self.sess.opts.unstable_opts.next_solver.map_or(false, |c| c.coherence)
+        self.sess.opts.unstable_opts.next_solver.coherence
     }
 
     pub fn is_impl_trait_in_trait(self, def_id: DefId) -> bool {
@@ -3183,8 +3183,7 @@ impl<'tcx> TyCtxt<'tcx> {
 
     /// Whether this is a trait implementation that has `#[diagnostic::do_not_recommend]`
     pub fn do_not_recommend_impl(self, def_id: DefId) -> bool {
-        matches!(self.def_kind(def_id), DefKind::Impl { of_trait: true })
-            && self.impl_trait_header(def_id).is_some_and(|header| header.do_not_recommend)
+        self.get_diagnostic_attr(def_id, sym::do_not_recommend).is_some()
     }
 }
 
diff --git a/compiler/rustc_middle/src/ty/fast_reject.rs b/compiler/rustc_middle/src/ty/fast_reject.rs
index 91344c4e39c..2945a0be424 100644
--- a/compiler/rustc_middle/src/ty/fast_reject.rs
+++ b/compiler/rustc_middle/src/ty/fast_reject.rs
@@ -3,6 +3,14 @@ pub use rustc_type_ir::fast_reject::*;
 
 use super::TyCtxt;
 
-pub type DeepRejectCtxt<'tcx> = rustc_type_ir::fast_reject::DeepRejectCtxt<TyCtxt<'tcx>>;
+pub type DeepRejectCtxt<
+    'tcx,
+    const INSTANTIATE_LHS_WITH_INFER: bool,
+    const INSTANTIATE_RHS_WITH_INFER: bool,
+> = rustc_type_ir::fast_reject::DeepRejectCtxt<
+    TyCtxt<'tcx>,
+    INSTANTIATE_LHS_WITH_INFER,
+    INSTANTIATE_RHS_WITH_INFER,
+>;
 
 pub type SimplifiedType = rustc_type_ir::fast_reject::SimplifiedType<DefId>;
diff --git a/compiler/rustc_middle/src/ty/impls_ty.rs b/compiler/rustc_middle/src/ty/impls_ty.rs
index b5b7b8bcfef..5f6305bb48a 100644
--- a/compiler/rustc_middle/src/ty/impls_ty.rs
+++ b/compiler/rustc_middle/src/ty/impls_ty.rs
@@ -75,11 +75,9 @@ impl<'a> HashStable<StableHashingContext<'a>> for mir::interpret::AllocId {
     }
 }
 
-// CtfeProvenance is an AllocId and a bool.
 impl<'a> HashStable<StableHashingContext<'a>> for mir::interpret::CtfeProvenance {
     fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
-        self.alloc_id().hash_stable(hcx, hasher);
-        self.immutable().hash_stable(hcx, hasher);
+        self.into_parts().hash_stable(hcx, hasher);
     }
 }
 
diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs
index d0a9039441d..8cec8eac189 100644
--- a/compiler/rustc_middle/src/ty/layout.rs
+++ b/compiler/rustc_middle/src/ty/layout.rs
@@ -1075,11 +1075,13 @@ where
                 // the raw pointer, so size and align are set to the boxed type, but `pointee.safe`
                 // will still be `None`.
                 if let Some(ref mut pointee) = result {
-                    if offset.bytes() == 0 && this.ty.is_box() {
+                    if offset.bytes() == 0
+                        && let Some(boxed_ty) = this.ty.boxed_ty()
+                    {
                         debug_assert!(pointee.safe.is_none());
                         let optimize = tcx.sess.opts.optimize != OptLevel::No;
                         pointee.safe = Some(PointerKind::Box {
-                            unpin: optimize && this.ty.boxed_ty().is_unpin(tcx, cx.param_env()),
+                            unpin: optimize && boxed_ty.is_unpin(tcx, cx.param_env()),
                             global: this.ty.is_box_global(tcx),
                         });
                     }
diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs
index ee70a6346d9..e637ced7139 100644
--- a/compiler/rustc_middle/src/ty/mod.rs
+++ b/compiler/rustc_middle/src/ty/mod.rs
@@ -263,7 +263,6 @@ pub struct ImplTraitHeader<'tcx> {
     pub trait_ref: ty::EarlyBinder<'tcx, ty::TraitRef<'tcx>>,
     pub polarity: ImplPolarity,
     pub safety: hir::Safety,
-    pub do_not_recommend: bool,
 }
 
 #[derive(Copy, Clone, PartialEq, Eq, Debug, TypeFoldable, TypeVisitable)]
@@ -1797,6 +1796,37 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
+    /// Get an attribute from the diagnostic attribute namespace
+    ///
+    /// This function requests an attribute with the following structure:
+    ///
+    /// `#[diagnostic::$attr]`
+    ///
+    /// This function performs feature checking, so if an attribute is returned
+    /// it can be used by the consumer
+    pub fn get_diagnostic_attr(
+        self,
+        did: impl Into<DefId>,
+        attr: Symbol,
+    ) -> Option<&'tcx ast::Attribute> {
+        let did: DefId = did.into();
+        if did.as_local().is_some() {
+            // it's a crate local item, we need to check feature flags
+            if rustc_feature::is_stable_diagnostic_attribute(attr, self.features()) {
+                self.get_attrs_by_path(did, &[sym::diagnostic, sym::do_not_recommend]).next()
+            } else {
+                None
+            }
+        } else {
+            // we filter out unstable diagnostic attributes before
+            // encoding attributes
+            debug_assert!(rustc_feature::encode_cross_crate(attr));
+            self.item_attrs(did)
+                .iter()
+                .find(|a| matches!(a.path().as_ref(), [sym::diagnostic, a] if *a == attr))
+        }
+    }
+
     pub fn get_attrs_by_path<'attr>(
         self,
         did: DefId,
diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs
index 89ef30fa768..730ba265b19 100644
--- a/compiler/rustc_middle/src/ty/sty.rs
+++ b/compiler/rustc_middle/src/ty/sty.rs
@@ -1136,6 +1136,7 @@ impl<'tcx> Ty<'tcx> {
     }
 
     /// Tests if this is any kind of primitive pointer type (reference, raw pointer, fn pointer).
+    /// `Box` is *not* considered a pointer here!
     #[inline]
     pub fn is_any_ptr(self) -> bool {
         self.is_ref() || self.is_unsafe_ptr() || self.is_fn_ptr()
@@ -1170,14 +1171,19 @@ impl<'tcx> Ty<'tcx> {
         }
     }
 
-    /// Panics if called on any type other than `Box<T>`.
-    pub fn boxed_ty(self) -> Ty<'tcx> {
+    pub fn boxed_ty(self) -> Option<Ty<'tcx>> {
         match self.kind() {
-            Adt(def, args) if def.is_box() => args.type_at(0),
-            _ => bug!("`boxed_ty` is called on non-box type {:?}", self),
+            Adt(def, args) if def.is_box() => Some(args.type_at(0)),
+            _ => None,
         }
     }
 
+    /// Panics if called on any type other than `Box<T>`.
+    pub fn expect_boxed_ty(self) -> Ty<'tcx> {
+        self.boxed_ty()
+            .unwrap_or_else(|| bug!("`expect_boxed_ty` is called on non-box type {:?}", self))
+    }
+
     /// A scalar type is one that denotes an atomic datum, with no sub-components.
     /// (A RawPtr is scalar because it represents a non-managed pointer, so its
     /// contents are abstract to rustc.)
@@ -1323,7 +1329,7 @@ impl<'tcx> Ty<'tcx> {
     /// Some types -- notably unsafe ptrs -- can only be dereferenced explicitly.
     pub fn builtin_deref(self, explicit: bool) -> Option<Ty<'tcx>> {
         match *self.kind() {
-            Adt(def, _) if def.is_box() => Some(self.boxed_ty()),
+            _ if let Some(boxed) = self.boxed_ty() => Some(boxed),
             Ref(_, ty, _) => Some(ty),
             RawPtr(ty, _) if explicit => Some(ty),
             _ => None,
diff --git a/compiler/rustc_middle/src/ty/trait_def.rs b/compiler/rustc_middle/src/ty/trait_def.rs
index dfb137f738f..82690f70e5f 100644
--- a/compiler/rustc_middle/src/ty/trait_def.rs
+++ b/compiler/rustc_middle/src/ty/trait_def.rs
@@ -168,9 +168,9 @@ impl<'tcx> TyCtxt<'tcx> {
         // whose outer level is not a parameter or projection. Especially for things like
         // `T: Clone` this is incredibly useful as we would otherwise look at all the impls
         // of `Clone` for `Option<T>`, `Vec<T>`, `ConcreteType` and so on.
-        // Note that we're using `TreatParams::ForLookup` to query `non_blanket_impls` while using
-        // `TreatParams::AsCandidateKey` while actually adding them.
-        if let Some(simp) = fast_reject::simplify_type(self, self_ty, TreatParams::ForLookup) {
+        // Note that we're using `TreatParams::AsRigid` to query `non_blanket_impls` while using
+        // `TreatParams::InstantiateWithInfer` while actually adding them.
+        if let Some(simp) = fast_reject::simplify_type(self, self_ty, TreatParams::AsRigid) {
             if let Some(impls) = impls.non_blanket_impls.get(&simp) {
                 for &impl_def_id in impls {
                     f(impl_def_id);
@@ -190,7 +190,9 @@ impl<'tcx> TyCtxt<'tcx> {
         self_ty: Ty<'tcx>,
     ) -> impl Iterator<Item = DefId> + 'tcx {
         let impls = self.trait_impls_of(trait_def_id);
-        if let Some(simp) = fast_reject::simplify_type(self, self_ty, TreatParams::AsCandidateKey) {
+        if let Some(simp) =
+            fast_reject::simplify_type(self, self_ty, TreatParams::InstantiateWithInfer)
+        {
             if let Some(impls) = impls.non_blanket_impls.get(&simp) {
                 return impls.iter().copied();
             }
@@ -239,7 +241,7 @@ pub(super) fn trait_impls_of_provider(tcx: TyCtxt<'_>, trait_id: DefId) -> Trait
         let impl_self_ty = tcx.type_of(impl_def_id).instantiate_identity();
 
         if let Some(simplified_self_ty) =
-            fast_reject::simplify_type(tcx, impl_self_ty, TreatParams::AsCandidateKey)
+            fast_reject::simplify_type(tcx, impl_self_ty, TreatParams::InstantiateWithInfer)
         {
             impls.non_blanket_impls.entry(simplified_self_ty).or_default().push(impl_def_id);
         } else {
diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index efbccca77c1..d70ff8258d0 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -1628,7 +1628,7 @@ impl<'tcx> ExplicitSelf<'tcx> {
             _ if is_self_ty(self_arg_ty) => ByValue,
             ty::Ref(region, ty, mutbl) if is_self_ty(ty) => ByReference(region, mutbl),
             ty::RawPtr(ty, mutbl) if is_self_ty(ty) => ByRawPointer(mutbl),
-            ty::Adt(def, _) if def.is_box() && is_self_ty(self_arg_ty.boxed_ty()) => ByBox,
+            _ if self_arg_ty.boxed_ty().is_some_and(is_self_ty) => ByBox,
             _ => Other,
         }
     }
diff --git a/compiler/rustc_mir_dataflow/src/impls/initialized.rs b/compiler/rustc_mir_dataflow/src/impls/initialized.rs
index ddfd0739358..a88927427ba 100644
--- a/compiler/rustc_mir_dataflow/src/impls/initialized.rs
+++ b/compiler/rustc_mir_dataflow/src/impls/initialized.rs
@@ -51,15 +51,15 @@ use crate::{
 /// Similarly, at a given `drop` statement, the set-intersection
 /// between this data and `MaybeUninitializedPlaces` yields the set of
 /// places that would require a dynamic drop-flag at that statement.
-pub struct MaybeInitializedPlaces<'a, 'mir, 'tcx> {
+pub struct MaybeInitializedPlaces<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
-    body: &'mir Body<'tcx>,
+    body: &'a Body<'tcx>,
     move_data: &'a MoveData<'tcx>,
     skip_unreachable_unwind: bool,
 }
 
-impl<'a, 'mir, 'tcx> MaybeInitializedPlaces<'a, 'mir, 'tcx> {
-    pub fn new(tcx: TyCtxt<'tcx>, body: &'mir Body<'tcx>, move_data: &'a MoveData<'tcx>) -> Self {
+impl<'a, 'tcx> MaybeInitializedPlaces<'a, 'tcx> {
+    pub fn new(tcx: TyCtxt<'tcx>, body: &'a Body<'tcx>, move_data: &'a MoveData<'tcx>) -> Self {
         MaybeInitializedPlaces { tcx, body, move_data, skip_unreachable_unwind: false }
     }
 
@@ -85,7 +85,7 @@ impl<'a, 'mir, 'tcx> MaybeInitializedPlaces<'a, 'mir, 'tcx> {
     }
 }
 
-impl<'a, 'mir, 'tcx> HasMoveData<'tcx> for MaybeInitializedPlaces<'a, 'mir, 'tcx> {
+impl<'a, 'tcx> HasMoveData<'tcx> for MaybeInitializedPlaces<'a, 'tcx> {
     fn move_data(&self) -> &MoveData<'tcx> {
         self.move_data
     }
@@ -126,17 +126,17 @@ impl<'a, 'mir, 'tcx> HasMoveData<'tcx> for MaybeInitializedPlaces<'a, 'mir, 'tcx
 /// Similarly, at a given `drop` statement, the set-intersection
 /// between this data and `MaybeInitializedPlaces` yields the set of
 /// places that would require a dynamic drop-flag at that statement.
-pub struct MaybeUninitializedPlaces<'a, 'mir, 'tcx> {
+pub struct MaybeUninitializedPlaces<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
-    body: &'mir Body<'tcx>,
+    body: &'a Body<'tcx>,
     move_data: &'a MoveData<'tcx>,
 
     mark_inactive_variants_as_uninit: bool,
     skip_unreachable_unwind: BitSet<mir::BasicBlock>,
 }
 
-impl<'a, 'mir, 'tcx> MaybeUninitializedPlaces<'a, 'mir, 'tcx> {
-    pub fn new(tcx: TyCtxt<'tcx>, body: &'mir Body<'tcx>, move_data: &'a MoveData<'tcx>) -> Self {
+impl<'a, 'tcx> MaybeUninitializedPlaces<'a, 'tcx> {
+    pub fn new(tcx: TyCtxt<'tcx>, body: &'a Body<'tcx>, move_data: &'a MoveData<'tcx>) -> Self {
         MaybeUninitializedPlaces {
             tcx,
             body,
@@ -165,7 +165,7 @@ impl<'a, 'mir, 'tcx> MaybeUninitializedPlaces<'a, 'mir, 'tcx> {
     }
 }
 
-impl<'a, 'tcx> HasMoveData<'tcx> for MaybeUninitializedPlaces<'a, '_, 'tcx> {
+impl<'tcx> HasMoveData<'tcx> for MaybeUninitializedPlaces<'_, 'tcx> {
     fn move_data(&self) -> &MoveData<'tcx> {
         self.move_data
     }
@@ -251,24 +251,24 @@ impl<'a, 'tcx> HasMoveData<'tcx> for DefinitelyInitializedPlaces<'a, 'tcx> {
 ///     c = S;                                  // {a, b, c, d }
 /// }
 /// ```
-pub struct EverInitializedPlaces<'a, 'mir, 'tcx> {
-    body: &'mir Body<'tcx>,
+pub struct EverInitializedPlaces<'a, 'tcx> {
+    body: &'a Body<'tcx>,
     move_data: &'a MoveData<'tcx>,
 }
 
-impl<'a, 'mir, 'tcx> EverInitializedPlaces<'a, 'mir, 'tcx> {
-    pub fn new(body: &'mir Body<'tcx>, move_data: &'a MoveData<'tcx>) -> Self {
+impl<'a, 'tcx> EverInitializedPlaces<'a, 'tcx> {
+    pub fn new(body: &'a Body<'tcx>, move_data: &'a MoveData<'tcx>) -> Self {
         EverInitializedPlaces { body, move_data }
     }
 }
 
-impl<'a, 'tcx> HasMoveData<'tcx> for EverInitializedPlaces<'a, '_, 'tcx> {
+impl<'tcx> HasMoveData<'tcx> for EverInitializedPlaces<'_, 'tcx> {
     fn move_data(&self) -> &MoveData<'tcx> {
         self.move_data
     }
 }
 
-impl<'a, 'mir, 'tcx> MaybeInitializedPlaces<'a, 'mir, 'tcx> {
+impl<'a, 'tcx> MaybeInitializedPlaces<'a, 'tcx> {
     fn update_bits(
         trans: &mut impl GenKill<MovePathIndex>,
         path: MovePathIndex,
@@ -281,7 +281,7 @@ impl<'a, 'mir, 'tcx> MaybeInitializedPlaces<'a, 'mir, 'tcx> {
     }
 }
 
-impl<'a, 'tcx> MaybeUninitializedPlaces<'a, '_, 'tcx> {
+impl<'tcx> MaybeUninitializedPlaces<'_, 'tcx> {
     fn update_bits(
         trans: &mut impl GenKill<MovePathIndex>,
         path: MovePathIndex,
@@ -307,7 +307,7 @@ impl<'a, 'tcx> DefinitelyInitializedPlaces<'a, 'tcx> {
     }
 }
 
-impl<'tcx> AnalysisDomain<'tcx> for MaybeInitializedPlaces<'_, '_, 'tcx> {
+impl<'tcx> AnalysisDomain<'tcx> for MaybeInitializedPlaces<'_, 'tcx> {
     /// There can be many more `MovePathIndex` than there are locals in a MIR body.
     /// We use a chunked bitset to avoid paying too high a memory footprint.
     type Domain = MaybeReachable<ChunkedBitSet<MovePathIndex>>;
@@ -329,7 +329,7 @@ impl<'tcx> AnalysisDomain<'tcx> for MaybeInitializedPlaces<'_, '_, 'tcx> {
     }
 }
 
-impl<'tcx> GenKillAnalysis<'tcx> for MaybeInitializedPlaces<'_, '_, 'tcx> {
+impl<'tcx> GenKillAnalysis<'tcx> for MaybeInitializedPlaces<'_, 'tcx> {
     type Idx = MovePathIndex;
 
     fn domain_size(&self, _: &Body<'tcx>) -> usize {
@@ -442,7 +442,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for MaybeInitializedPlaces<'_, '_, 'tcx> {
     }
 }
 
-impl<'tcx> AnalysisDomain<'tcx> for MaybeUninitializedPlaces<'_, '_, 'tcx> {
+impl<'tcx> AnalysisDomain<'tcx> for MaybeUninitializedPlaces<'_, 'tcx> {
     /// There can be many more `MovePathIndex` than there are locals in a MIR body.
     /// We use a chunked bitset to avoid paying too high a memory footprint.
     type Domain = ChunkedBitSet<MovePathIndex>;
@@ -466,7 +466,7 @@ impl<'tcx> AnalysisDomain<'tcx> for MaybeUninitializedPlaces<'_, '_, 'tcx> {
     }
 }
 
-impl<'tcx> GenKillAnalysis<'tcx> for MaybeUninitializedPlaces<'_, '_, 'tcx> {
+impl<'tcx> GenKillAnalysis<'tcx> for MaybeUninitializedPlaces<'_, 'tcx> {
     type Idx = MovePathIndex;
 
     fn domain_size(&self, _: &Body<'tcx>) -> usize {
@@ -643,7 +643,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for DefinitelyInitializedPlaces<'_, 'tcx> {
     }
 }
 
-impl<'tcx> AnalysisDomain<'tcx> for EverInitializedPlaces<'_, '_, 'tcx> {
+impl<'tcx> AnalysisDomain<'tcx> for EverInitializedPlaces<'_, 'tcx> {
     /// There can be many more `InitIndex` than there are locals in a MIR body.
     /// We use a chunked bitset to avoid paying too high a memory footprint.
     type Domain = ChunkedBitSet<InitIndex>;
@@ -662,7 +662,7 @@ impl<'tcx> AnalysisDomain<'tcx> for EverInitializedPlaces<'_, '_, 'tcx> {
     }
 }
 
-impl<'tcx> GenKillAnalysis<'tcx> for EverInitializedPlaces<'_, '_, 'tcx> {
+impl<'tcx> GenKillAnalysis<'tcx> for EverInitializedPlaces<'_, 'tcx> {
     type Idx = InitIndex;
 
     fn domain_size(&self, _: &Body<'tcx>) -> usize {
diff --git a/compiler/rustc_mir_dataflow/src/move_paths/builder.rs b/compiler/rustc_mir_dataflow/src/move_paths/builder.rs
index 14390723ba4..87ea7299238 100644
--- a/compiler/rustc_mir_dataflow/src/move_paths/builder.rs
+++ b/compiler/rustc_mir_dataflow/src/move_paths/builder.rs
@@ -16,6 +16,7 @@ use super::{
 
 struct MoveDataBuilder<'a, 'tcx, F> {
     body: &'a Body<'tcx>,
+    loc: Location,
     tcx: TyCtxt<'tcx>,
     param_env: ty::ParamEnv<'tcx>,
     data: MoveData<'tcx>,
@@ -56,6 +57,7 @@ impl<'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> MoveDataBuilder<'a, 'tcx, F> {
 
         MoveDataBuilder {
             body,
+            loc: Location::START,
             tcx,
             param_env,
             data: MoveData {
@@ -107,7 +109,7 @@ enum MovePathResult {
     Error,
 }
 
-impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> {
+impl<'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> MoveDataBuilder<'a, 'tcx, F> {
     /// This creates a MovePath for a given place, returning an `MovePathError`
     /// if that place can't be moved from.
     ///
@@ -116,7 +118,7 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> {
     ///
     /// Maybe we should have separate "borrowck" and "moveck" modes.
     fn move_path_for(&mut self, place: Place<'tcx>) -> MovePathResult {
-        let data = &mut self.builder.data;
+        let data = &mut self.data;
 
         debug!("lookup({:?})", place);
         let Some(mut base) = data.rev_lookup.find_local(place.local) else {
@@ -131,8 +133,8 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> {
         let mut union_path = None;
 
         for (place_ref, elem) in data.rev_lookup.un_derefer.iter_projections(place.as_ref()) {
-            let body = self.builder.body;
-            let tcx = self.builder.tcx;
+            let body = self.body;
+            let tcx = self.tcx;
             let place_ty = place_ref.ty(body, tcx).ty;
             if place_ty.references_error() {
                 return MovePathResult::Error;
@@ -238,7 +240,7 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> {
                 | ProjectionElem::Downcast(_, _) => (),
             }
             let elem_ty = PlaceTy::from_ty(place_ty).projection_ty(tcx, elem).ty;
-            if !(self.builder.filter)(elem_ty) {
+            if !(self.filter)(elem_ty) {
                 return MovePathResult::Error;
             }
             if union_path.is_none() {
@@ -274,7 +276,7 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> {
             data: MoveData { rev_lookup, move_paths, path_map, init_path_map, .. },
             tcx,
             ..
-        } = self.builder;
+        } = self;
         *rev_lookup.projections.entry((base, elem.lift())).or_insert_with(move || {
             new_move_path(move_paths, path_map, init_path_map, Some(base), mk_place(*tcx))
         })
@@ -285,9 +287,7 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> {
         // drop), so this not being a valid move path is OK.
         let _ = self.move_path_for(place);
     }
-}
 
-impl<'a, 'tcx, F> MoveDataBuilder<'a, 'tcx, F> {
     fn finalize(self) -> MoveData<'tcx> {
         debug!("{}", {
             debug!("moves for {:?}:", self.body.span);
@@ -317,12 +317,12 @@ pub(super) fn gather_moves<'tcx>(
 
     for (bb, block) in body.basic_blocks.iter_enumerated() {
         for (i, stmt) in block.statements.iter().enumerate() {
-            let source = Location { block: bb, statement_index: i };
-            builder.gather_statement(source, stmt);
+            builder.loc = Location { block: bb, statement_index: i };
+            builder.gather_statement(stmt);
         }
 
-        let terminator_loc = Location { block: bb, statement_index: block.statements.len() };
-        builder.gather_terminator(terminator_loc, block.terminator());
+        builder.loc = Location { block: bb, statement_index: block.statements.len() };
+        builder.gather_terminator(block.terminator());
     }
 
     builder.finalize()
@@ -345,30 +345,14 @@ impl<'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> MoveDataBuilder<'a, 'tcx, F> {
         }
     }
 
-    fn gather_statement(&mut self, loc: Location, stmt: &Statement<'tcx>) {
-        debug!("gather_statement({:?}, {:?})", loc, stmt);
-        (Gatherer { builder: self, loc }).gather_statement(stmt);
-    }
-
-    fn gather_terminator(&mut self, loc: Location, term: &Terminator<'tcx>) {
-        debug!("gather_terminator({:?}, {:?})", loc, term);
-        (Gatherer { builder: self, loc }).gather_terminator(term);
-    }
-}
-
-struct Gatherer<'b, 'a, 'tcx, F> {
-    builder: &'b mut MoveDataBuilder<'a, 'tcx, F>,
-    loc: Location,
-}
-
-impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> {
     fn gather_statement(&mut self, stmt: &Statement<'tcx>) {
+        debug!("gather_statement({:?}, {:?})", self.loc, stmt);
         match &stmt.kind {
             StatementKind::Assign(box (place, Rvalue::CopyForDeref(reffed))) => {
                 let local = place.as_local().unwrap();
-                assert!(self.builder.body.local_decls[local].is_deref_temp());
+                assert!(self.body.local_decls[local].is_deref_temp());
 
-                let rev_lookup = &mut self.builder.data.rev_lookup;
+                let rev_lookup = &mut self.data.rev_lookup;
 
                 rev_lookup.un_derefer.insert(local, reffed.as_ref());
                 let base_local = rev_lookup.un_derefer.deref_chain(local).first().unwrap().local;
@@ -380,7 +364,7 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> {
                     // Box starts out uninitialized - need to create a separate
                     // move-path for the interior so it will be separate from
                     // the exterior.
-                    self.create_move_path(self.builder.tcx.mk_place_deref(*place));
+                    self.create_move_path(self.tcx.mk_place_deref(*place));
                     self.gather_init(place.as_ref(), InitKind::Shallow);
                 } else {
                     self.gather_init(place.as_ref(), InitKind::Deep);
@@ -393,7 +377,7 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> {
             StatementKind::StorageLive(_) => {}
             StatementKind::StorageDead(local) => {
                 // DerefTemp locals (results of CopyForDeref) don't actually move anything.
-                if !self.builder.body.local_decls[*local].is_deref_temp() {
+                if !self.body.local_decls[*local].is_deref_temp() {
                     self.gather_move(Place::from(*local));
                 }
             }
@@ -443,6 +427,7 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> {
     }
 
     fn gather_terminator(&mut self, term: &Terminator<'tcx>) {
+        debug!("gather_terminator({:?}, {:?})", self.loc, term);
         match term.kind {
             TerminatorKind::Goto { target: _ }
             | TerminatorKind::FalseEdge { .. }
@@ -551,7 +536,7 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> {
             // `ConstIndex` patterns. This is done to ensure that all move paths
             // are disjoint, which is expected by drop elaboration.
             let base_place =
-                Place { local: place.local, projection: self.builder.tcx.mk_place_elems(base) };
+                Place { local: place.local, projection: self.tcx.mk_place_elems(base) };
             let base_path = match self.move_path_for(base_place) {
                 MovePathResult::Path(path) => path,
                 MovePathResult::Union(path) => {
@@ -562,11 +547,9 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> {
                     return;
                 }
             };
-            let base_ty = base_place.ty(self.builder.body, self.builder.tcx).ty;
+            let base_ty = base_place.ty(self.body, self.tcx).ty;
             let len: u64 = match base_ty.kind() {
-                ty::Array(_, size) => {
-                    size.eval_target_usize(self.builder.tcx, self.builder.param_env)
-                }
+                ty::Array(_, size) => size.eval_target_usize(self.tcx, self.param_env),
                 _ => bug!("from_end: false slice pattern of non-array type"),
             };
             for offset in from..to {
@@ -587,13 +570,13 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> {
     }
 
     fn record_move(&mut self, place: Place<'tcx>, path: MovePathIndex) {
-        let move_out = self.builder.data.moves.push(MoveOut { path, source: self.loc });
+        let move_out = self.data.moves.push(MoveOut { path, source: self.loc });
         debug!(
             "gather_move({:?}, {:?}): adding move {:?} of {:?}",
             self.loc, place, move_out, path
         );
-        self.builder.data.path_map[path].push(move_out);
-        self.builder.data.loc_map[self.loc].push(move_out);
+        self.data.path_map[path].push(move_out);
+        self.data.loc_map[self.loc].push(move_out);
     }
 
     fn gather_init(&mut self, place: PlaceRef<'tcx>, kind: InitKind) {
@@ -604,13 +587,13 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> {
         // Check if we are assigning into a field of a union, if so, lookup the place
         // of the union so it is marked as initialized again.
         if let Some((place_base, ProjectionElem::Field(_, _))) = place.last_projection() {
-            if place_base.ty(self.builder.body, self.builder.tcx).ty.is_union() {
+            if place_base.ty(self.body, self.tcx).ty.is_union() {
                 place = place_base;
             }
         }
 
-        if let LookupResult::Exact(path) = self.builder.data.rev_lookup.find(place) {
-            let init = self.builder.data.inits.push(Init {
+        if let LookupResult::Exact(path) = self.data.rev_lookup.find(place) {
+            let init = self.data.inits.push(Init {
                 location: InitLocation::Statement(self.loc),
                 path,
                 kind,
@@ -621,8 +604,8 @@ impl<'b, 'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> Gatherer<'b, 'a, 'tcx, F> {
                 self.loc, place, init, path
             );
 
-            self.builder.data.init_path_map[path].push(init);
-            self.builder.data.init_loc_map[self.loc].push(init);
+            self.data.init_path_map[path].push(init);
+            self.data.init_loc_map[self.loc].push(init);
         }
     }
 }
diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs
index 2b20a35b61e..05404366320 100644
--- a/compiler/rustc_mir_dataflow/src/value_analysis.rs
+++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs
@@ -923,14 +923,14 @@ impl<'tcx> Map<'tcx> {
     }
 }
 
-struct PlaceCollector<'a, 'b, 'tcx> {
+struct PlaceCollector<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
-    body: &'b Body<'tcx>,
+    body: &'a Body<'tcx>,
     map: &'a mut Map<'tcx>,
     assignments: FxIndexSet<(PlaceIndex, PlaceIndex)>,
 }
 
-impl<'tcx> PlaceCollector<'_, '_, 'tcx> {
+impl<'tcx> PlaceCollector<'_, 'tcx> {
     #[tracing::instrument(level = "trace", skip(self))]
     fn register_place(&mut self, place: Place<'tcx>) -> Option<PlaceIndex> {
         // Create a place for this projection.
@@ -967,7 +967,7 @@ impl<'tcx> PlaceCollector<'_, '_, 'tcx> {
     }
 }
 
-impl<'tcx> Visitor<'tcx> for PlaceCollector<'_, '_, 'tcx> {
+impl<'tcx> Visitor<'tcx> for PlaceCollector<'_, 'tcx> {
     #[tracing::instrument(level = "trace", skip(self))]
     fn visit_place(&mut self, place: &Place<'tcx>, ctxt: PlaceContext, _: Location) {
         if !ctxt.is_use() {
diff --git a/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs b/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs
index e4bc6b3efe4..8291df9be53 100644
--- a/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs
+++ b/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs
@@ -20,7 +20,7 @@ use rustc_target::spec::PanicStrategy;
 /// This forces all unwinds, in panic=abort mode happening in foreign code, to
 /// trigger a process abort.
 #[derive(PartialEq)]
-pub struct AbortUnwindingCalls;
+pub(super) struct AbortUnwindingCalls;
 
 impl<'tcx> crate::MirPass<'tcx> for AbortUnwindingCalls {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
@@ -50,9 +50,7 @@ impl<'tcx> crate::MirPass<'tcx> for AbortUnwindingCalls {
         // with a function call, and whose function we're calling may unwind.
         // This will filter to functions with `extern "C-unwind"` ABIs, for
         // example.
-        let mut calls_to_terminate = Vec::new();
-        let mut cleanups_to_remove = Vec::new();
-        for (id, block) in body.basic_blocks.iter_enumerated() {
+        for block in body.basic_blocks.as_mut() {
             if block.is_cleanup {
                 continue;
             }
@@ -61,7 +59,7 @@ impl<'tcx> crate::MirPass<'tcx> for AbortUnwindingCalls {
 
             let call_can_unwind = match &terminator.kind {
                 TerminatorKind::Call { func, .. } => {
-                    let ty = func.ty(body, tcx);
+                    let ty = func.ty(&body.local_decls, tcx);
                     let sig = ty.fn_sig(tcx);
                     let fn_def_id = match ty.kind() {
                         ty::FnPtr(..) => None,
@@ -86,33 +84,22 @@ impl<'tcx> crate::MirPass<'tcx> for AbortUnwindingCalls {
                 _ => continue,
             };
 
-            // If this function call can't unwind, then there's no need for it
-            // to have a landing pad. This means that we can remove any cleanup
-            // registered for it.
             if !call_can_unwind {
-                cleanups_to_remove.push(id);
-                continue;
-            }
-
-            // Otherwise if this function can unwind, then if the outer function
-            // can also unwind there's nothing to do. If the outer function
-            // can't unwind, however, we need to change the landing pad for this
-            // function call to one that aborts.
-            if !body_can_unwind {
-                calls_to_terminate.push(id);
+                // If this function call can't unwind, then there's no need for it
+                // to have a landing pad. This means that we can remove any cleanup
+                // registered for it.
+                let cleanup = block.terminator_mut().unwind_mut().unwrap();
+                *cleanup = UnwindAction::Unreachable;
+            } else if !body_can_unwind {
+                // Otherwise if this function can unwind, then if the outer function
+                // can also unwind there's nothing to do. If the outer function
+                // can't unwind, however, we need to change the landing pad for this
+                // function call to one that aborts.
+                let cleanup = block.terminator_mut().unwind_mut().unwrap();
+                *cleanup = UnwindAction::Terminate(UnwindTerminateReason::Abi);
             }
         }
 
-        for id in calls_to_terminate {
-            let cleanup = body.basic_blocks_mut()[id].terminator_mut().unwind_mut().unwrap();
-            *cleanup = UnwindAction::Terminate(UnwindTerminateReason::Abi);
-        }
-
-        for id in cleanups_to_remove {
-            let cleanup = body.basic_blocks_mut()[id].terminator_mut().unwind_mut().unwrap();
-            *cleanup = UnwindAction::Unreachable;
-        }
-
         // We may have invalidated some `cleanup` blocks so clean those up now.
         super::simplify::remove_dead_blocks(body);
     }
diff --git a/compiler/rustc_mir_transform/src/add_call_guards.rs b/compiler/rustc_mir_transform/src/add_call_guards.rs
index 78e850de3c7..18a0746f54f 100644
--- a/compiler/rustc_mir_transform/src/add_call_guards.rs
+++ b/compiler/rustc_mir_transform/src/add_call_guards.rs
@@ -4,11 +4,11 @@ use rustc_middle::ty::TyCtxt;
 use tracing::debug;
 
 #[derive(PartialEq)]
-pub enum AddCallGuards {
+pub(super) enum AddCallGuards {
     AllCallEdges,
     CriticalCallEdges,
 }
-pub use self::AddCallGuards::*;
+pub(super) use self::AddCallGuards::*;
 
 /**
  * Breaks outgoing critical edges for call terminators in the MIR.
@@ -37,7 +37,7 @@ impl<'tcx> crate::MirPass<'tcx> for AddCallGuards {
 }
 
 impl AddCallGuards {
-    pub fn add_call_guards(&self, body: &mut Body<'_>) {
+    pub(super) fn add_call_guards(&self, body: &mut Body<'_>) {
         let mut pred_count: IndexVec<_, _> =
             body.basic_blocks.predecessors().iter().map(|ps| ps.len()).collect();
         pred_count[START_BLOCK] += 1;
diff --git a/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs b/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs
index 4a8196aeff5..47572d8d3b2 100644
--- a/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs
+++ b/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs
@@ -35,7 +35,7 @@ use crate::util;
 ///
 /// The storage instructions are required to avoid stack space
 /// blowup.
-pub struct AddMovesForPackedDrops;
+pub(super) struct AddMovesForPackedDrops;
 
 impl<'tcx> crate::MirPass<'tcx> for AddMovesForPackedDrops {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
@@ -44,7 +44,7 @@ impl<'tcx> crate::MirPass<'tcx> for AddMovesForPackedDrops {
     }
 }
 
-pub fn add_moves_for_packed_drops<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+fn add_moves_for_packed_drops<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
     let patch = add_moves_for_packed_drops_patch(tcx, body);
     patch.apply(body);
 }
diff --git a/compiler/rustc_mir_transform/src/add_retag.rs b/compiler/rustc_mir_transform/src/add_retag.rs
index 2e12064fe73..794e85fbfed 100644
--- a/compiler/rustc_mir_transform/src/add_retag.rs
+++ b/compiler/rustc_mir_transform/src/add_retag.rs
@@ -8,7 +8,7 @@ use rustc_hir::LangItem;
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 
-pub struct AddRetag;
+pub(super) struct AddRetag;
 
 /// Determine whether this type may contain a reference (or box), and thus needs retagging.
 /// We will only recurse `depth` times into Tuples/ADTs to bound the cost of this.
diff --git a/compiler/rustc_mir_transform/src/add_subtyping_projections.rs b/compiler/rustc_mir_transform/src/add_subtyping_projections.rs
index 369f6c60084..ab6bf18b30c 100644
--- a/compiler/rustc_mir_transform/src/add_subtyping_projections.rs
+++ b/compiler/rustc_mir_transform/src/add_subtyping_projections.rs
@@ -1,15 +1,14 @@
-use rustc_index::IndexVec;
 use rustc_middle::mir::patch::MirPatch;
 use rustc_middle::mir::visit::MutVisitor;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
 
-pub struct Subtyper;
+pub(super) struct Subtyper;
 
-pub struct SubTypeChecker<'a, 'tcx> {
+struct SubTypeChecker<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
     patcher: MirPatch<'tcx>,
-    local_decls: &'a IndexVec<Local, LocalDecl<'tcx>>,
+    local_decls: &'a LocalDecls<'tcx>,
 }
 
 impl<'a, 'tcx> MutVisitor<'tcx> for SubTypeChecker<'a, 'tcx> {
@@ -52,7 +51,7 @@ impl<'a, 'tcx> MutVisitor<'tcx> for SubTypeChecker<'a, 'tcx> {
 // // gets transformed to
 // let temp: rval_ty = rval;
 // let place: place_ty = temp as place_ty;
-pub fn subtype_finder<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+fn subtype_finder<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
     let patch = MirPatch::new(body);
     let mut checker = SubTypeChecker { tcx, patcher: patch, local_decls: &body.local_decls };
 
diff --git a/compiler/rustc_mir_transform/src/check_alignment.rs b/compiler/rustc_mir_transform/src/check_alignment.rs
index 2e072aa262a..e1d5152ae51 100644
--- a/compiler/rustc_mir_transform/src/check_alignment.rs
+++ b/compiler/rustc_mir_transform/src/check_alignment.rs
@@ -7,7 +7,7 @@ use rustc_middle::ty::{self, ParamEnv, Ty, TyCtxt};
 use rustc_session::Session;
 use tracing::{debug, trace};
 
-pub struct CheckAlignment;
+pub(super) struct CheckAlignment;
 
 impl<'tcx> crate::MirPass<'tcx> for CheckAlignment {
     fn is_enabled(&self, sess: &Session) -> bool {
diff --git a/compiler/rustc_mir_transform/src/check_const_item_mutation.rs b/compiler/rustc_mir_transform/src/check_const_item_mutation.rs
index fb03323e37e..234ed8206f5 100644
--- a/compiler/rustc_mir_transform/src/check_const_item_mutation.rs
+++ b/compiler/rustc_mir_transform/src/check_const_item_mutation.rs
@@ -8,7 +8,7 @@ use rustc_span::Span;
 
 use crate::errors;
 
-pub struct CheckConstItemMutation;
+pub(super) struct CheckConstItemMutation;
 
 impl<'tcx> crate::MirLint<'tcx> for CheckConstItemMutation {
     fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) {
diff --git a/compiler/rustc_mir_transform/src/check_packed_ref.rs b/compiler/rustc_mir_transform/src/check_packed_ref.rs
index 2f957de7e78..1922d4fef25 100644
--- a/compiler/rustc_mir_transform/src/check_packed_ref.rs
+++ b/compiler/rustc_mir_transform/src/check_packed_ref.rs
@@ -5,7 +5,7 @@ use rustc_middle::ty::{self, TyCtxt};
 
 use crate::{errors, util};
 
-pub struct CheckPackedRef;
+pub(super) struct CheckPackedRef;
 
 impl<'tcx> crate::MirLint<'tcx> for CheckPackedRef {
     fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) {
diff --git a/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs b/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs
index 2f3be1e425d..97ce16c0661 100644
--- a/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs
+++ b/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs
@@ -21,7 +21,7 @@ use rustc_middle::mir::{Body, BorrowKind, CastKind, Rvalue, StatementKind, Termi
 use rustc_middle::ty::adjustment::PointerCoercion;
 use rustc_middle::ty::TyCtxt;
 
-pub struct CleanupPostBorrowck;
+pub(super) struct CleanupPostBorrowck;
 
 impl<'tcx> crate::MirPass<'tcx> for CleanupPostBorrowck {
     fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
diff --git a/compiler/rustc_mir_transform/src/copy_prop.rs b/compiler/rustc_mir_transform/src/copy_prop.rs
index 85d25ca2231..dc4ee58ea83 100644
--- a/compiler/rustc_mir_transform/src/copy_prop.rs
+++ b/compiler/rustc_mir_transform/src/copy_prop.rs
@@ -17,7 +17,7 @@ use crate::ssa::SsaLocals;
 /// where each of the locals is only assigned once.
 ///
 /// We want to replace all those locals by `_a`, either copied or moved.
-pub struct CopyProp;
+pub(super) struct CopyProp;
 
 impl<'tcx> crate::MirPass<'tcx> for CopyProp {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs
index eefb748e49d..90243cd2910 100644
--- a/compiler/rustc_mir_transform/src/coroutine.rs
+++ b/compiler/rustc_mir_transform/src/coroutine.rs
@@ -53,7 +53,7 @@
 mod by_move_body;
 use std::{iter, ops};
 
-pub use by_move_body::coroutine_by_move_body_def_id;
+pub(super) use by_move_body::coroutine_by_move_body_def_id;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::pluralize;
 use rustc_hir as hir;
@@ -85,7 +85,7 @@ use tracing::{debug, instrument, trace};
 use crate::deref_separator::deref_finder;
 use crate::{abort_unwinding_calls, errors, pass_manager as pm, simplify};
 
-pub struct StateTransform;
+pub(super) struct StateTransform;
 
 struct RenameLocalVisitor<'tcx> {
     from: Local,
@@ -872,9 +872,9 @@ fn compute_storage_conflicts<'mir, 'tcx>(
     storage_conflicts
 }
 
-struct StorageConflictVisitor<'mir, 'tcx, 's> {
-    body: &'mir Body<'tcx>,
-    saved_locals: &'s CoroutineSavedLocals,
+struct StorageConflictVisitor<'a, 'tcx> {
+    body: &'a Body<'tcx>,
+    saved_locals: &'a CoroutineSavedLocals,
     // FIXME(tmandry): Consider using sparse bitsets here once we have good
     // benchmarks for coroutines.
     local_conflicts: BitMatrix<Local, Local>,
@@ -882,8 +882,8 @@ struct StorageConflictVisitor<'mir, 'tcx, 's> {
     eligible_storage_live: BitSet<Local>,
 }
 
-impl<'mir, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'mir, 'tcx, R>
-    for StorageConflictVisitor<'mir, 'tcx, '_>
+impl<'a, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'a, 'tcx, R>
+    for StorageConflictVisitor<'a, 'tcx>
 {
     type FlowState = BitSet<Local>;
 
@@ -891,7 +891,7 @@ impl<'mir, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'mir, 'tcx, R>
         &mut self,
         _results: &mut R,
         state: &Self::FlowState,
-        _statement: &'mir Statement<'tcx>,
+        _statement: &'a Statement<'tcx>,
         loc: Location,
     ) {
         self.apply_state(state, loc);
@@ -901,14 +901,14 @@ impl<'mir, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'mir, 'tcx, R>
         &mut self,
         _results: &mut R,
         state: &Self::FlowState,
-        _terminator: &'mir Terminator<'tcx>,
+        _terminator: &'a Terminator<'tcx>,
         loc: Location,
     ) {
         self.apply_state(state, loc);
     }
 }
 
-impl StorageConflictVisitor<'_, '_, '_> {
+impl StorageConflictVisitor<'_, '_> {
     fn apply_state(&mut self, flow_state: &BitSet<Local>, loc: Location) {
         // Ignore unreachable blocks.
         if let TerminatorKind::Unreachable = self.body.basic_blocks[loc.block].terminator().kind {
@@ -1199,7 +1199,7 @@ fn insert_panic_block<'tcx>(
     message: AssertMessage<'tcx>,
 ) -> BasicBlock {
     let assert_block = BasicBlock::new(body.basic_blocks.len());
-    let term = TerminatorKind::Assert {
+    let kind = TerminatorKind::Assert {
         cond: Operand::Constant(Box::new(ConstOperand {
             span: body.span,
             user_ty: None,
@@ -1211,14 +1211,7 @@ fn insert_panic_block<'tcx>(
         unwind: UnwindAction::Continue,
     };
 
-    let source_info = SourceInfo::outermost(body.span);
-    body.basic_blocks_mut().push(BasicBlockData {
-        statements: Vec::new(),
-        terminator: Some(Terminator { source_info, kind: term }),
-        is_cleanup: false,
-    });
-
-    assert_block
+    insert_term_block(body, kind)
 }
 
 fn can_return<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, param_env: ty::ParamEnv<'tcx>) -> bool {
diff --git a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs
index cf39c136b01..6476f3c6238 100644
--- a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs
+++ b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs
@@ -82,12 +82,17 @@ use rustc_middle::ty::{self, InstanceKind, Ty, TyCtxt, TypeVisitableExt};
 use rustc_span::symbol::kw;
 use rustc_target::abi::{FieldIdx, VariantIdx};
 
-pub fn coroutine_by_move_body_def_id<'tcx>(
+pub(crate) fn coroutine_by_move_body_def_id<'tcx>(
     tcx: TyCtxt<'tcx>,
     coroutine_def_id: LocalDefId,
 ) -> DefId {
     let body = tcx.mir_built(coroutine_def_id).borrow();
 
+    // If the typeck results are tainted, no need to make a by-ref body.
+    if body.tainted_by_errors.is_some() {
+        return coroutine_def_id.to_def_id();
+    }
+
     let Some(hir::CoroutineKind::Desugared(_, hir::CoroutineSource::Closure)) =
         tcx.coroutine_kind(coroutine_def_id)
     else {
@@ -98,7 +103,9 @@ pub fn coroutine_by_move_body_def_id<'tcx>(
     // the MIR body will be constructed well.
     let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty;
 
-    let ty::Coroutine(_, args) = *coroutine_ty.kind() else { bug!("{body:#?}") };
+    let ty::Coroutine(_, args) = *coroutine_ty.kind() else {
+        bug!("tried to create by-move body of non-coroutine receiver");
+    };
     let args = args.as_coroutine();
 
     let coroutine_kind = args.kind_ty().to_opt_closure_kind().unwrap();
@@ -107,7 +114,7 @@ pub fn coroutine_by_move_body_def_id<'tcx>(
     let ty::CoroutineClosure(_, parent_args) =
         *tcx.type_of(parent_def_id).instantiate_identity().kind()
     else {
-        bug!();
+        bug!("coroutine's parent was not a coroutine-closure");
     };
     if parent_args.references_error() {
         return coroutine_def_id.to_def_id();
diff --git a/compiler/rustc_mir_transform/src/cost_checker.rs b/compiler/rustc_mir_transform/src/cost_checker.rs
index a1c1422912e..59b403538a3 100644
--- a/compiler/rustc_mir_transform/src/cost_checker.rs
+++ b/compiler/rustc_mir_transform/src/cost_checker.rs
@@ -12,7 +12,7 @@ const CONST_SWITCH_BONUS: usize = 10;
 
 /// Verify that the callee body is compatible with the caller.
 #[derive(Clone)]
-pub(crate) struct CostChecker<'b, 'tcx> {
+pub(super) struct CostChecker<'b, 'tcx> {
     tcx: TyCtxt<'tcx>,
     param_env: ParamEnv<'tcx>,
     penalty: usize,
@@ -22,7 +22,7 @@ pub(crate) struct CostChecker<'b, 'tcx> {
 }
 
 impl<'b, 'tcx> CostChecker<'b, 'tcx> {
-    pub fn new(
+    pub(super) fn new(
         tcx: TyCtxt<'tcx>,
         param_env: ParamEnv<'tcx>,
         instance: Option<ty::Instance<'tcx>>,
@@ -36,7 +36,7 @@ impl<'b, 'tcx> CostChecker<'b, 'tcx> {
     /// Needed because the `CostChecker` is used sometimes for just blocks,
     /// and even the full `Inline` doesn't call `visit_body`, so there's nowhere
     /// to put this logic in the visitor.
-    pub fn add_function_level_costs(&mut self) {
+    pub(super) fn add_function_level_costs(&mut self) {
         fn is_call_like(bbd: &BasicBlockData<'_>) -> bool {
             use TerminatorKind::*;
             match bbd.terminator().kind {
@@ -64,7 +64,7 @@ impl<'b, 'tcx> CostChecker<'b, 'tcx> {
         }
     }
 
-    pub fn cost(&self) -> usize {
+    pub(super) fn cost(&self) -> usize {
         usize::saturating_sub(self.penalty, self.bonus)
     }
 
diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs
index 4edba61fdec..042f589768a 100644
--- a/compiler/rustc_mir_transform/src/coverage/mod.rs
+++ b/compiler/rustc_mir_transform/src/coverage/mod.rs
@@ -1,4 +1,4 @@
-pub mod query;
+pub(super) mod query;
 
 mod counters;
 mod graph;
@@ -32,7 +32,7 @@ use crate::coverage::mappings::ExtractedMappings;
 /// Inserts `StatementKind::Coverage` statements that either instrument the binary with injected
 /// counters, via intrinsic `llvm.instrprof.increment`, and/or inject metadata used during codegen
 /// to construct the coverage map.
-pub struct InstrumentCoverage;
+pub(super) struct InstrumentCoverage;
 
 impl<'tcx> crate::MirPass<'tcx> for InstrumentCoverage {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs
index fcc774503f4..b904b0d2599 100644
--- a/compiler/rustc_mir_transform/src/coverage/spans.rs
+++ b/compiler/rustc_mir_transform/src/coverage/spans.rs
@@ -3,7 +3,7 @@ use std::collections::VecDeque;
 use rustc_data_structures::captures::Captures;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_middle::mir;
-use rustc_span::Span;
+use rustc_span::{DesugaringKind, ExpnKind, MacroKind, Span};
 use tracing::{debug, debug_span, instrument};
 
 use crate::coverage::graph::{BasicCoverageBlock, CoverageGraph};
@@ -25,7 +25,7 @@ pub(super) fn extract_refined_covspans(
 
     // First, perform the passes that need macro information.
     covspans.sort_by(|a, b| basic_coverage_blocks.cmp_in_dominator_order(a.bcb, b.bcb));
-    remove_unwanted_macro_spans(&mut covspans);
+    remove_unwanted_expansion_spans(&mut covspans);
     split_visible_macro_spans(&mut covspans);
 
     // We no longer need the extra information in `SpanFromMir`, so convert to `Covspan`.
@@ -76,18 +76,24 @@ pub(super) fn extract_refined_covspans(
 /// invocation, which is unhelpful. Keeping only the first such span seems to
 /// give better mappings, so remove the others.
 ///
+/// Similarly, `await` expands to a branch on the discriminant of `Poll`, which
+/// leads to incorrect coverage if the `Future` is immediately ready (#98712).
+///
 /// (The input spans should be sorted in BCB dominator order, so that the
 /// retained "first" span is likely to dominate the others.)
-fn remove_unwanted_macro_spans(covspans: &mut Vec<SpanFromMir>) {
-    let mut seen_macro_spans = FxHashSet::default();
+fn remove_unwanted_expansion_spans(covspans: &mut Vec<SpanFromMir>) {
+    let mut deduplicated_spans = FxHashSet::default();
+
     covspans.retain(|covspan| {
-        // Ignore (retain) non-macro-expansion spans.
-        if covspan.visible_macro.is_none() {
-            return true;
+        match covspan.expn_kind {
+            // Retain only the first await-related or macro-expanded covspan with this span.
+            Some(ExpnKind::Desugaring(kind)) if kind == DesugaringKind::Await => {
+                deduplicated_spans.insert(covspan.span)
+            }
+            Some(ExpnKind::Macro(MacroKind::Bang, _)) => deduplicated_spans.insert(covspan.span),
+            // Ignore (retain) other spans.
+            _ => true,
         }
-
-        // Retain only the first macro-expanded covspan with this span.
-        seen_macro_spans.insert(covspan.span)
     });
 }
 
@@ -99,7 +105,9 @@ fn split_visible_macro_spans(covspans: &mut Vec<SpanFromMir>) {
     let mut extra_spans = vec![];
 
     covspans.retain(|covspan| {
-        let Some(visible_macro) = covspan.visible_macro else { return true };
+        let Some(ExpnKind::Macro(MacroKind::Bang, visible_macro)) = covspan.expn_kind else {
+            return true;
+        };
 
         let split_len = visible_macro.as_str().len() as u32 + 1;
         let (before, after) = covspan.span.split_at(split_len);
@@ -111,8 +119,8 @@ fn split_visible_macro_spans(covspans: &mut Vec<SpanFromMir>) {
             return true;
         }
 
-        extra_spans.push(SpanFromMir::new(before, covspan.visible_macro, covspan.bcb));
-        extra_spans.push(SpanFromMir::new(after, covspan.visible_macro, covspan.bcb));
+        extra_spans.push(SpanFromMir::new(before, covspan.expn_kind.clone(), covspan.bcb));
+        extra_spans.push(SpanFromMir::new(after, covspan.expn_kind.clone(), covspan.bcb));
         false // Discard the original covspan that we just split.
     });
 
diff --git a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs
index 32bd25bf4b9..7f5765c9462 100644
--- a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs
+++ b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs
@@ -3,13 +3,13 @@ use rustc_middle::mir::coverage::CoverageKind;
 use rustc_middle::mir::{
     self, FakeReadCause, Statement, StatementKind, Terminator, TerminatorKind,
 };
-use rustc_span::{Span, Symbol};
+use rustc_span::{ExpnKind, Span};
 
 use crate::coverage::graph::{
     BasicCoverageBlock, BasicCoverageBlockData, CoverageGraph, START_BCB,
 };
 use crate::coverage::spans::Covspan;
-use crate::coverage::unexpand::unexpand_into_body_span_with_visible_macro;
+use crate::coverage::unexpand::unexpand_into_body_span_with_expn_kind;
 use crate::coverage::ExtractedHirInfo;
 
 pub(crate) struct ExtractedCovspans {
@@ -60,7 +60,7 @@ fn bcb_to_initial_coverage_spans<'a, 'tcx>(
         let data = &mir_body[bb];
 
         let unexpand = move |expn_span| {
-            unexpand_into_body_span_with_visible_macro(expn_span, body_span)
+            unexpand_into_body_span_with_expn_kind(expn_span, body_span)
                 // Discard any spans that fill the entire body, because they tend
                 // to represent compiler-inserted code, e.g. implicitly returning `()`.
                 .filter(|(span, _)| !span.source_equal(body_span))
@@ -68,9 +68,9 @@ fn bcb_to_initial_coverage_spans<'a, 'tcx>(
 
         let mut extract_statement_span = |statement| {
             let expn_span = filtered_statement_span(statement)?;
-            let (span, visible_macro) = unexpand(expn_span)?;
+            let (span, expn_kind) = unexpand(expn_span)?;
 
-            initial_covspans.push(SpanFromMir::new(span, visible_macro, bcb));
+            initial_covspans.push(SpanFromMir::new(span, expn_kind, bcb));
             Some(())
         };
         for statement in data.statements.iter() {
@@ -79,9 +79,9 @@ fn bcb_to_initial_coverage_spans<'a, 'tcx>(
 
         let mut extract_terminator_span = |terminator| {
             let expn_span = filtered_terminator_span(terminator)?;
-            let (span, visible_macro) = unexpand(expn_span)?;
+            let (span, expn_kind) = unexpand(expn_span)?;
 
-            initial_covspans.push(SpanFromMir::new(span, visible_macro, bcb));
+            initial_covspans.push(SpanFromMir::new(span, expn_kind, bcb));
             Some(())
         };
         extract_terminator_span(data.terminator());
@@ -214,7 +214,7 @@ pub(crate) struct SpanFromMir {
     /// With the exception of `fn_sig_span`, this should always be contained
     /// within `body_span`.
     pub(crate) span: Span,
-    pub(crate) visible_macro: Option<Symbol>,
+    pub(crate) expn_kind: Option<ExpnKind>,
     pub(crate) bcb: BasicCoverageBlock,
 }
 
@@ -223,12 +223,12 @@ impl SpanFromMir {
         Self::new(fn_sig_span, None, START_BCB)
     }
 
-    pub(crate) fn new(span: Span, visible_macro: Option<Symbol>, bcb: BasicCoverageBlock) -> Self {
-        Self { span, visible_macro, bcb }
+    pub(crate) fn new(span: Span, expn_kind: Option<ExpnKind>, bcb: BasicCoverageBlock) -> Self {
+        Self { span, expn_kind, bcb }
     }
 
     pub(crate) fn into_covspan(self) -> Covspan {
-        let Self { span, visible_macro: _, bcb } = self;
+        let Self { span, expn_kind: _, bcb } = self;
         Covspan { span, bcb }
     }
 }
diff --git a/compiler/rustc_mir_transform/src/coverage/unexpand.rs b/compiler/rustc_mir_transform/src/coverage/unexpand.rs
index 8cde291b907..cb861544736 100644
--- a/compiler/rustc_mir_transform/src/coverage/unexpand.rs
+++ b/compiler/rustc_mir_transform/src/coverage/unexpand.rs
@@ -1,4 +1,4 @@
-use rustc_span::{ExpnKind, MacroKind, Span, Symbol};
+use rustc_span::{ExpnKind, Span};
 
 /// Walks through the expansion ancestors of `original_span` to find a span that
 /// is contained in `body_span` and has the same [syntax context] as `body_span`.
@@ -13,20 +13,15 @@ pub(crate) fn unexpand_into_body_span(original_span: Span, body_span: Span) -> O
 ///
 /// If the returned span represents a bang-macro invocation (e.g. `foo!(..)`),
 /// the returned symbol will be the name of that macro (e.g. `foo`).
-pub(crate) fn unexpand_into_body_span_with_visible_macro(
+pub(crate) fn unexpand_into_body_span_with_expn_kind(
     original_span: Span,
     body_span: Span,
-) -> Option<(Span, Option<Symbol>)> {
+) -> Option<(Span, Option<ExpnKind>)> {
     let (span, prev) = unexpand_into_body_span_with_prev(original_span, body_span)?;
 
-    let visible_macro = prev
-        .map(|prev| match prev.ctxt().outer_expn_data().kind {
-            ExpnKind::Macro(MacroKind::Bang, name) => Some(name),
-            _ => None,
-        })
-        .flatten();
+    let expn_kind = prev.map(|prev| prev.ctxt().outer_expn_data().kind);
 
-    Some((span, visible_macro))
+    Some((span, expn_kind))
 }
 
 /// Walks through the expansion ancestors of `original_span` to find a span that
diff --git a/compiler/rustc_mir_transform/src/cross_crate_inline.rs b/compiler/rustc_mir_transform/src/cross_crate_inline.rs
index 50aaed090f6..ce109ef7674 100644
--- a/compiler/rustc_mir_transform/src/cross_crate_inline.rs
+++ b/compiler/rustc_mir_transform/src/cross_crate_inline.rs
@@ -10,7 +10,7 @@ use rustc_span::sym;
 
 use crate::{inline, pass_manager as pm};
 
-pub fn provide(providers: &mut Providers) {
+pub(super) fn provide(providers: &mut Providers) {
     providers.cross_crate_inlinable = cross_crate_inlinable;
 }
 
diff --git a/compiler/rustc_mir_transform/src/ctfe_limit.rs b/compiler/rustc_mir_transform/src/ctfe_limit.rs
index ea473b64ce5..bd58b1b6689 100644
--- a/compiler/rustc_mir_transform/src/ctfe_limit.rs
+++ b/compiler/rustc_mir_transform/src/ctfe_limit.rs
@@ -8,7 +8,7 @@ use rustc_middle::mir::{
 use rustc_middle::ty::TyCtxt;
 use tracing::instrument;
 
-pub struct CtfeLimit;
+pub(super) struct CtfeLimit;
 
 impl<'tcx> crate::MirPass<'tcx> for CtfeLimit {
     #[instrument(skip(self, _tcx, body))]
diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
index 46f7408ef80..79f12be4bc3 100644
--- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
+++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
@@ -26,7 +26,7 @@ use tracing::{debug, debug_span, instrument};
 const BLOCK_LIMIT: usize = 100;
 const PLACE_LIMIT: usize = 100;
 
-pub struct DataflowConstProp;
+pub(super) struct DataflowConstProp;
 
 impl<'tcx> crate::MirPass<'tcx> for DataflowConstProp {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
@@ -332,7 +332,7 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> {
 }
 
 impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> {
-    pub fn new(tcx: TyCtxt<'tcx>, body: &'a Body<'tcx>, map: Map<'tcx>) -> Self {
+    fn new(tcx: TyCtxt<'tcx>, body: &'a Body<'tcx>, map: Map<'tcx>) -> Self {
         let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id());
         Self {
             map,
@@ -838,14 +838,14 @@ impl<'tcx> MutVisitor<'tcx> for Patch<'tcx> {
     }
 }
 
-struct OperandCollector<'tcx, 'map, 'locals, 'a> {
+struct OperandCollector<'a, 'locals, 'tcx> {
     state: &'a State<FlatSet<Scalar>>,
     visitor: &'a mut Collector<'tcx, 'locals>,
-    ecx: &'map mut InterpCx<'tcx, DummyMachine>,
-    map: &'map Map<'tcx>,
+    ecx: &'a mut InterpCx<'tcx, DummyMachine>,
+    map: &'a Map<'tcx>,
 }
 
-impl<'tcx> Visitor<'tcx> for OperandCollector<'tcx, '_, '_, '_> {
+impl<'tcx> Visitor<'tcx> for OperandCollector<'_, '_, 'tcx> {
     fn visit_projection_elem(
         &mut self,
         _: PlaceRef<'tcx>,
diff --git a/compiler/rustc_mir_transform/src/dead_store_elimination.rs b/compiler/rustc_mir_transform/src/dead_store_elimination.rs
index 9081a2e2e30..65c037559c5 100644
--- a/compiler/rustc_mir_transform/src/dead_store_elimination.rs
+++ b/compiler/rustc_mir_transform/src/dead_store_elimination.rs
@@ -28,7 +28,7 @@ use crate::util::is_within_packed;
 ///
 /// The `borrowed` set must be a `BitSet` of all the locals that are ever borrowed in this body. It
 /// can be generated via the [`borrowed_locals`] function.
-pub fn eliminate<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+fn eliminate<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
     let borrowed_locals = borrowed_locals(body);
 
     // If the user requests complete debuginfo, mark the locals that appear in it as live, so
@@ -127,7 +127,7 @@ pub fn eliminate<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
     }
 }
 
-pub enum DeadStoreElimination {
+pub(super) enum DeadStoreElimination {
     Initial,
     Final,
 }
diff --git a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
index 71af099199e..e870a71b05a 100644
--- a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
+++ b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
@@ -150,7 +150,7 @@ fn type_will_always_be_passed_directly(ty: Ty<'_>) -> bool {
 /// body of the function instead of just the signature. These can be useful for optimization
 /// purposes on a best-effort basis. We compute them here and store them into the crate metadata so
 /// dependent crates can use them.
-pub fn deduced_param_attrs<'tcx>(
+pub(super) fn deduced_param_attrs<'tcx>(
     tcx: TyCtxt<'tcx>,
     def_id: LocalDefId,
 ) -> &'tcx [DeducedParamAttrs] {
diff --git a/compiler/rustc_mir_transform/src/deduplicate_blocks.rs b/compiler/rustc_mir_transform/src/deduplicate_blocks.rs
index be50c1da8a4..c8179d513c7 100644
--- a/compiler/rustc_mir_transform/src/deduplicate_blocks.rs
+++ b/compiler/rustc_mir_transform/src/deduplicate_blocks.rs
@@ -13,7 +13,7 @@ use tracing::debug;
 
 use super::simplify::simplify_cfg;
 
-pub struct DeduplicateBlocks;
+pub(super) struct DeduplicateBlocks;
 
 impl<'tcx> crate::MirPass<'tcx> for DeduplicateBlocks {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
diff --git a/compiler/rustc_mir_transform/src/deref_separator.rs b/compiler/rustc_mir_transform/src/deref_separator.rs
index a878f777448..ad7ccef4976 100644
--- a/compiler/rustc_mir_transform/src/deref_separator.rs
+++ b/compiler/rustc_mir_transform/src/deref_separator.rs
@@ -1,16 +1,15 @@
-use rustc_index::IndexVec;
 use rustc_middle::mir::patch::MirPatch;
 use rustc_middle::mir::visit::NonUseContext::VarDebugInfo;
 use rustc_middle::mir::visit::{MutVisitor, PlaceContext};
 use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
 
-pub struct Derefer;
+pub(super) struct Derefer;
 
-pub struct DerefChecker<'a, 'tcx> {
+struct DerefChecker<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
     patcher: MirPatch<'tcx>,
-    local_decls: &'a IndexVec<Local, LocalDecl<'tcx>>,
+    local_decls: &'a LocalDecls<'tcx>,
 }
 
 impl<'a, 'tcx> MutVisitor<'tcx> for DerefChecker<'a, 'tcx> {
@@ -67,7 +66,7 @@ impl<'a, 'tcx> MutVisitor<'tcx> for DerefChecker<'a, 'tcx> {
     }
 }
 
-pub fn deref_finder<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+pub(super) fn deref_finder<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
     let patch = MirPatch::new(body);
     let mut checker = DerefChecker { tcx, patcher: patch, local_decls: &body.local_decls };
 
diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs
index 67bee36b8a5..175097e30ee 100644
--- a/compiler/rustc_mir_transform/src/dest_prop.rs
+++ b/compiler/rustc_mir_transform/src/dest_prop.rs
@@ -146,7 +146,7 @@ use rustc_mir_dataflow::points::{save_as_intervals, DenseLocationMap, PointIndex
 use rustc_mir_dataflow::Analysis;
 use tracing::{debug, trace};
 
-pub struct DestinationPropagation;
+pub(super) struct DestinationPropagation;
 
 impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
diff --git a/compiler/rustc_mir_transform/src/dump_mir.rs b/compiler/rustc_mir_transform/src/dump_mir.rs
index 06ae1b490d7..1640d34d6c8 100644
--- a/compiler/rustc_mir_transform/src/dump_mir.rs
+++ b/compiler/rustc_mir_transform/src/dump_mir.rs
@@ -7,7 +7,7 @@ use rustc_middle::mir::{write_mir_pretty, Body};
 use rustc_middle::ty::TyCtxt;
 use rustc_session::config::{OutFileName, OutputType};
 
-pub struct Marker(pub &'static str);
+pub(super) struct Marker(pub &'static str);
 
 impl<'tcx> crate::MirPass<'tcx> for Marker {
     fn name(&self) -> &'static str {
diff --git a/compiler/rustc_mir_transform/src/early_otherwise_branch.rs b/compiler/rustc_mir_transform/src/early_otherwise_branch.rs
index 1c54cd70023..3884321df2a 100644
--- a/compiler/rustc_mir_transform/src/early_otherwise_branch.rs
+++ b/compiler/rustc_mir_transform/src/early_otherwise_branch.rs
@@ -90,7 +90,7 @@ use super::simplify::simplify_cfg;
 ///     |      ...      |
 ///     =================
 /// ```
-pub struct EarlyOtherwiseBranch;
+pub(super) struct EarlyOtherwiseBranch;
 
 impl<'tcx> crate::MirPass<'tcx> for EarlyOtherwiseBranch {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
@@ -310,42 +310,28 @@ fn verify_candidate_branch<'tcx>(
 ) -> bool {
     // In order for the optimization to be correct, the branch must...
     // ...have exactly one statement
-    let [statement] = branch.statements.as_slice() else {
-        return false;
-    };
-    // ...assign the discriminant of `place` in that statement
-    let StatementKind::Assign(boxed) = &statement.kind else { return false };
-    let (discr_place, Rvalue::Discriminant(from_place)) = &**boxed else { return false };
-    if *from_place != place {
-        return false;
-    }
-    // ...make that assignment to a local
-    if discr_place.projection.len() != 0 {
-        return false;
-    }
-    // ...terminate on a `SwitchInt` that invalidates that local
-    let TerminatorKind::SwitchInt { discr: switch_op, targets, .. } = &branch.terminator().kind
-    else {
-        return false;
-    };
-    if *switch_op != Operand::Move(*discr_place) {
-        return false;
-    }
-    // ...fall through to `destination` if the switch misses
-    if destination != targets.otherwise() {
-        return false;
-    }
-    // ...have a branch for value `value`
-    let mut iter = targets.iter();
-    let Some((target_value, _)) = iter.next() else {
-        return false;
-    };
-    if target_value != value {
-        return false;
-    }
-    // ...and have no more branches
-    if let Some(_) = iter.next() {
-        return false;
+    if let [statement] = branch.statements.as_slice()
+        // ...assign the discriminant of `place` in that statement
+        && let StatementKind::Assign(boxed) = &statement.kind
+        && let (discr_place, Rvalue::Discriminant(from_place)) = &**boxed
+        && *from_place == place
+        // ...make that assignment to a local
+        && discr_place.projection.is_empty()
+        // ...terminate on a `SwitchInt` that invalidates that local
+        && let TerminatorKind::SwitchInt { discr: switch_op, targets, .. } =
+            &branch.terminator().kind
+        && *switch_op == Operand::Move(*discr_place)
+        // ...fall through to `destination` if the switch misses
+        && destination == targets.otherwise()
+        // ...have a branch for value `value`
+        && let mut iter = targets.iter()
+        && let Some((target_value, _)) = iter.next()
+        && target_value == value
+        // ...and have no more branches
+        && iter.next().is_none()
+    {
+        true
+    } else {
+        false
     }
-    true
 }
diff --git a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs
index 5dd82f40163..367a8c07593 100644
--- a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs
+++ b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs
@@ -11,7 +11,7 @@ use rustc_middle::ty::{Ty, TyCtxt};
 use rustc_target::abi::FieldIdx;
 
 /// Constructs the types used when accessing a Box's pointer
-pub fn build_ptr_tys<'tcx>(
+fn build_ptr_tys<'tcx>(
     tcx: TyCtxt<'tcx>,
     pointee: Ty<'tcx>,
     unique_did: DefId,
@@ -26,7 +26,7 @@ pub fn build_ptr_tys<'tcx>(
 }
 
 /// Constructs the projection needed to access a Box's pointer
-pub fn build_projection<'tcx>(
+pub(super) fn build_projection<'tcx>(
     unique_ty: Ty<'tcx>,
     nonnull_ty: Ty<'tcx>,
     ptr_ty: Ty<'tcx>,
@@ -62,11 +62,13 @@ impl<'tcx, 'a> MutVisitor<'tcx> for ElaborateBoxDerefVisitor<'tcx, 'a> {
         let base_ty = self.local_decls[place.local].ty;
 
         // Derefer ensures that derefs are always the first projection
-        if place.projection.first() == Some(&PlaceElem::Deref) && base_ty.is_box() {
+        if let Some(PlaceElem::Deref) = place.projection.first()
+            && let Some(boxed_ty) = base_ty.boxed_ty()
+        {
             let source_info = self.local_decls[place.local].source_info;
 
             let (unique_ty, nonnull_ty, ptr_ty) =
-                build_ptr_tys(tcx, base_ty.boxed_ty(), self.unique_did, self.nonnull_did);
+                build_ptr_tys(tcx, boxed_ty, self.unique_did, self.nonnull_did);
 
             let ptr_local = self.patch.new_temp(ptr_ty, source_info.span);
 
@@ -86,66 +88,65 @@ impl<'tcx, 'a> MutVisitor<'tcx> for ElaborateBoxDerefVisitor<'tcx, 'a> {
     }
 }
 
-pub struct ElaborateBoxDerefs;
+pub(super) struct ElaborateBoxDerefs;
 
 impl<'tcx> crate::MirPass<'tcx> for ElaborateBoxDerefs {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-        if let Some(def_id) = tcx.lang_items().owned_box() {
-            let unique_did = tcx.adt_def(def_id).non_enum_variant().fields[FieldIdx::ZERO].did;
+        // If box is not present, this pass doesn't need to do anything.
+        let Some(def_id) = tcx.lang_items().owned_box() else { return };
 
-            let Some(nonnull_def) = tcx.type_of(unique_did).instantiate_identity().ty_adt_def()
-            else {
-                span_bug!(tcx.def_span(unique_did), "expected Box to contain Unique")
-            };
+        let unique_did = tcx.adt_def(def_id).non_enum_variant().fields[FieldIdx::ZERO].did;
 
-            let nonnull_did = nonnull_def.non_enum_variant().fields[FieldIdx::ZERO].did;
+        let Some(nonnull_def) = tcx.type_of(unique_did).instantiate_identity().ty_adt_def() else {
+            span_bug!(tcx.def_span(unique_did), "expected Box to contain Unique")
+        };
 
-            let patch = MirPatch::new(body);
+        let nonnull_did = nonnull_def.non_enum_variant().fields[FieldIdx::ZERO].did;
 
-            let local_decls = &mut body.local_decls;
+        let patch = MirPatch::new(body);
 
-            let mut visitor =
-                ElaborateBoxDerefVisitor { tcx, unique_did, nonnull_did, local_decls, patch };
+        let local_decls = &mut body.local_decls;
 
-            for (block, data) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() {
-                visitor.visit_basic_block_data(block, data);
-            }
+        let mut visitor =
+            ElaborateBoxDerefVisitor { tcx, unique_did, nonnull_did, local_decls, patch };
 
-            visitor.patch.apply(body);
+        for (block, data) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() {
+            visitor.visit_basic_block_data(block, data);
+        }
 
-            for debug_info in body.var_debug_info.iter_mut() {
-                if let VarDebugInfoContents::Place(place) = &mut debug_info.value {
-                    let mut new_projections: Option<Vec<_>> = None;
+        visitor.patch.apply(body);
 
-                    for (base, elem) in place.iter_projections() {
-                        let base_ty = base.ty(&body.local_decls, tcx).ty;
+        for debug_info in body.var_debug_info.iter_mut() {
+            if let VarDebugInfoContents::Place(place) = &mut debug_info.value {
+                let mut new_projections: Option<Vec<_>> = None;
 
-                        if elem == PlaceElem::Deref && base_ty.is_box() {
-                            // Clone the projections before us, since now we need to mutate them.
-                            let new_projections =
-                                new_projections.get_or_insert_with(|| base.projection.to_vec());
+                for (base, elem) in place.iter_projections() {
+                    let base_ty = base.ty(&body.local_decls, tcx).ty;
 
-                            let (unique_ty, nonnull_ty, ptr_ty) =
-                                build_ptr_tys(tcx, base_ty.boxed_ty(), unique_did, nonnull_did);
+                    if let PlaceElem::Deref = elem
+                        && let Some(boxed_ty) = base_ty.boxed_ty()
+                    {
+                        // Clone the projections before us, since now we need to mutate them.
+                        let new_projections =
+                            new_projections.get_or_insert_with(|| base.projection.to_vec());
 
-                            new_projections.extend_from_slice(&build_projection(
-                                unique_ty, nonnull_ty, ptr_ty,
-                            ));
-                            new_projections.push(PlaceElem::Deref);
-                        } else if let Some(new_projections) = new_projections.as_mut() {
-                            // Keep building up our projections list once we've started it.
-                            new_projections.push(elem);
-                        }
-                    }
+                        let (unique_ty, nonnull_ty, ptr_ty) =
+                            build_ptr_tys(tcx, boxed_ty, unique_did, nonnull_did);
 
-                    // Store the mutated projections if we actually changed something.
-                    if let Some(new_projections) = new_projections {
-                        place.projection = tcx.mk_place_elems(&new_projections);
+                        new_projections
+                            .extend_from_slice(&build_projection(unique_ty, nonnull_ty, ptr_ty));
+                        new_projections.push(PlaceElem::Deref);
+                    } else if let Some(new_projections) = new_projections.as_mut() {
+                        // Keep building up our projections list once we've started it.
+                        new_projections.push(elem);
                     }
                 }
+
+                // Store the mutated projections if we actually changed something.
+                if let Some(new_projections) = new_projections {
+                    place.projection = tcx.mk_place_elems(&new_projections);
+                }
             }
-        } else {
-            // box is not present, this pass doesn't need to do anything
         }
     }
 }
diff --git a/compiler/rustc_mir_transform/src/elaborate_drops.rs b/compiler/rustc_mir_transform/src/elaborate_drops.rs
index f4a951ebde6..cb729792dc5 100644
--- a/compiler/rustc_mir_transform/src/elaborate_drops.rs
+++ b/compiler/rustc_mir_transform/src/elaborate_drops.rs
@@ -47,7 +47,7 @@ use crate::deref_separator::deref_finder;
 ///     }
 /// }
 /// ```
-pub struct ElaborateDrops;
+pub(super) struct ElaborateDrops;
 
 impl<'tcx> crate::MirPass<'tcx> for ElaborateDrops {
     #[instrument(level = "trace", skip(self, tcx, body))]
@@ -98,9 +98,9 @@ impl<'tcx> crate::MirPass<'tcx> for ElaborateDrops {
 /// Records unwind edges which are known to be unreachable, because they are in `drop` terminators
 /// that can't drop anything.
 #[instrument(level = "trace", skip(body, flow_inits), ret)]
-fn compute_dead_unwinds<'mir, 'tcx>(
-    body: &'mir Body<'tcx>,
-    flow_inits: &mut ResultsCursor<'mir, 'tcx, MaybeInitializedPlaces<'_, 'mir, 'tcx>>,
+fn compute_dead_unwinds<'a, 'tcx>(
+    body: &'a Body<'tcx>,
+    flow_inits: &mut ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>>,
 ) -> BitSet<BasicBlock> {
     // We only need to do this pass once, because unwind edges can only
     // reach cleanup blocks, which can't have unwind edges themselves.
@@ -121,12 +121,12 @@ fn compute_dead_unwinds<'mir, 'tcx>(
     dead_unwinds
 }
 
-struct InitializationData<'a, 'mir, 'tcx> {
-    inits: ResultsCursor<'mir, 'tcx, MaybeInitializedPlaces<'a, 'mir, 'tcx>>,
-    uninits: ResultsCursor<'mir, 'tcx, MaybeUninitializedPlaces<'a, 'mir, 'tcx>>,
+struct InitializationData<'a, 'tcx> {
+    inits: ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>>,
+    uninits: ResultsCursor<'a, 'tcx, MaybeUninitializedPlaces<'a, 'tcx>>,
 }
 
-impl InitializationData<'_, '_, '_> {
+impl InitializationData<'_, '_> {
     fn seek_before(&mut self, loc: Location) {
         self.inits.seek_before_primary_effect(loc);
         self.uninits.seek_before_primary_effect(loc);
@@ -137,45 +137,35 @@ impl InitializationData<'_, '_, '_> {
     }
 }
 
-struct Elaborator<'a, 'b, 'mir, 'tcx> {
-    ctxt: &'a mut ElaborateDropsCtxt<'b, 'mir, 'tcx>,
-}
-
-impl fmt::Debug for Elaborator<'_, '_, '_, '_> {
-    fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        Ok(())
-    }
-}
-
-impl<'a, 'tcx> DropElaborator<'a, 'tcx> for Elaborator<'a, '_, '_, 'tcx> {
+impl<'a, 'tcx> DropElaborator<'a, 'tcx> for ElaborateDropsCtxt<'a, 'tcx> {
     type Path = MovePathIndex;
 
     fn patch(&mut self) -> &mut MirPatch<'tcx> {
-        &mut self.ctxt.patch
+        &mut self.patch
     }
 
     fn body(&self) -> &'a Body<'tcx> {
-        self.ctxt.body
+        self.body
     }
 
     fn tcx(&self) -> TyCtxt<'tcx> {
-        self.ctxt.tcx
+        self.tcx
     }
 
     fn param_env(&self) -> ty::ParamEnv<'tcx> {
-        self.ctxt.param_env()
+        self.param_env()
     }
 
     #[instrument(level = "debug", skip(self), ret)]
     fn drop_style(&self, path: Self::Path, mode: DropFlagMode) -> DropStyle {
         let ((maybe_live, maybe_dead), multipart) = match mode {
-            DropFlagMode::Shallow => (self.ctxt.init_data.maybe_live_dead(path), false),
+            DropFlagMode::Shallow => (self.init_data.maybe_live_dead(path), false),
             DropFlagMode::Deep => {
                 let mut some_live = false;
                 let mut some_dead = false;
                 let mut children_count = 0;
-                on_all_children_bits(self.ctxt.move_data(), path, |child| {
-                    let (live, dead) = self.ctxt.init_data.maybe_live_dead(child);
+                on_all_children_bits(self.move_data(), path, |child| {
+                    let (live, dead) = self.init_data.maybe_live_dead(child);
                     debug!("elaborate_drop: state({:?}) = {:?}", child, (live, dead));
                     some_live |= live;
                     some_dead |= dead;
@@ -195,25 +185,25 @@ impl<'a, 'tcx> DropElaborator<'a, 'tcx> for Elaborator<'a, '_, '_, 'tcx> {
     fn clear_drop_flag(&mut self, loc: Location, path: Self::Path, mode: DropFlagMode) {
         match mode {
             DropFlagMode::Shallow => {
-                self.ctxt.set_drop_flag(loc, path, DropFlagState::Absent);
+                self.set_drop_flag(loc, path, DropFlagState::Absent);
             }
             DropFlagMode::Deep => {
-                on_all_children_bits(self.ctxt.move_data(), path, |child| {
-                    self.ctxt.set_drop_flag(loc, child, DropFlagState::Absent)
+                on_all_children_bits(self.move_data(), path, |child| {
+                    self.set_drop_flag(loc, child, DropFlagState::Absent)
                 });
             }
         }
     }
 
     fn field_subpath(&self, path: Self::Path, field: FieldIdx) -> Option<Self::Path> {
-        rustc_mir_dataflow::move_path_children_matching(self.ctxt.move_data(), path, |e| match e {
+        rustc_mir_dataflow::move_path_children_matching(self.move_data(), path, |e| match e {
             ProjectionElem::Field(idx, _) => idx == field,
             _ => false,
         })
     }
 
     fn array_subpath(&self, path: Self::Path, index: u64, size: u64) -> Option<Self::Path> {
-        rustc_mir_dataflow::move_path_children_matching(self.ctxt.move_data(), path, |e| match e {
+        rustc_mir_dataflow::move_path_children_matching(self.move_data(), path, |e| match e {
             ProjectionElem::ConstantIndex { offset, min_length, from_end } => {
                 debug_assert!(size == min_length, "min_length should be exact for arrays");
                 assert!(!from_end, "from_end should not be used for array element ConstantIndex");
@@ -224,34 +214,40 @@ impl<'a, 'tcx> DropElaborator<'a, 'tcx> for Elaborator<'a, '_, '_, 'tcx> {
     }
 
     fn deref_subpath(&self, path: Self::Path) -> Option<Self::Path> {
-        rustc_mir_dataflow::move_path_children_matching(self.ctxt.move_data(), path, |e| {
+        rustc_mir_dataflow::move_path_children_matching(self.move_data(), path, |e| {
             e == ProjectionElem::Deref
         })
     }
 
     fn downcast_subpath(&self, path: Self::Path, variant: VariantIdx) -> Option<Self::Path> {
-        rustc_mir_dataflow::move_path_children_matching(self.ctxt.move_data(), path, |e| match e {
+        rustc_mir_dataflow::move_path_children_matching(self.move_data(), path, |e| match e {
             ProjectionElem::Downcast(_, idx) => idx == variant,
             _ => false,
         })
     }
 
     fn get_drop_flag(&mut self, path: Self::Path) -> Option<Operand<'tcx>> {
-        self.ctxt.drop_flag(path).map(Operand::Copy)
+        self.drop_flag(path).map(Operand::Copy)
     }
 }
 
-struct ElaborateDropsCtxt<'a, 'mir, 'tcx> {
+struct ElaborateDropsCtxt<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
-    body: &'mir Body<'tcx>,
+    body: &'a Body<'tcx>,
     env: &'a MoveDataParamEnv<'tcx>,
-    init_data: InitializationData<'a, 'mir, 'tcx>,
+    init_data: InitializationData<'a, 'tcx>,
     drop_flags: IndexVec<MovePathIndex, Option<Local>>,
     patch: MirPatch<'tcx>,
 }
 
-impl<'b, 'mir, 'tcx> ElaborateDropsCtxt<'b, 'mir, 'tcx> {
-    fn move_data(&self) -> &'b MoveData<'tcx> {
+impl fmt::Debug for ElaborateDropsCtxt<'_, '_> {
+    fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        Ok(())
+    }
+}
+
+impl<'a, 'tcx> ElaborateDropsCtxt<'a, 'tcx> {
+    fn move_data(&self) -> &'a MoveData<'tcx> {
         &self.env.move_data
     }
 
@@ -370,15 +366,7 @@ impl<'b, 'mir, 'tcx> ElaborateDropsCtxt<'b, 'mir, 'tcx> {
                         }
                     };
                     self.init_data.seek_before(self.body.terminator_loc(bb));
-                    elaborate_drop(
-                        &mut Elaborator { ctxt: self },
-                        terminator.source_info,
-                        place,
-                        path,
-                        target,
-                        unwind,
-                        bb,
-                    )
+                    elaborate_drop(self, terminator.source_info, place, path, target, unwind, bb)
                 }
                 LookupResult::Parent(None) => {}
                 LookupResult::Parent(Some(_)) => {
diff --git a/compiler/rustc_mir_transform/src/errors.rs b/compiler/rustc_mir_transform/src/errors.rs
index 2703dc57cda..9a93c3a72b1 100644
--- a/compiler/rustc_mir_transform/src/errors.rs
+++ b/compiler/rustc_mir_transform/src/errors.rs
@@ -64,7 +64,7 @@ impl<'a, P: std::fmt::Debug> LintDiagnostic<'a, ()> for AssertLint<P> {
 }
 
 impl AssertLintKind {
-    pub fn lint(&self) -> &'static Lint {
+    pub(crate) fn lint(&self) -> &'static Lint {
         match self {
             AssertLintKind::ArithmeticOverflow => lint::builtin::ARITHMETIC_OVERFLOW,
             AssertLintKind::UnconditionalPanic => lint::builtin::UNCONDITIONAL_PANIC,
diff --git a/compiler/rustc_mir_transform/src/function_item_references.rs b/compiler/rustc_mir_transform/src/function_item_references.rs
index 199fd0f10ee..0efaa172e0c 100644
--- a/compiler/rustc_mir_transform/src/function_item_references.rs
+++ b/compiler/rustc_mir_transform/src/function_item_references.rs
@@ -11,7 +11,7 @@ use rustc_target::spec::abi::Abi;
 
 use crate::errors;
 
-pub struct FunctionItemReferences;
+pub(super) struct FunctionItemReferences;
 
 impl<'tcx> crate::MirLint<'tcx> for FunctionItemReferences {
     fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) {
diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index df0fcc42e59..e5d2b13efd8 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -109,7 +109,7 @@ use tracing::{debug, instrument, trace};
 
 use crate::ssa::{AssignedValue, SsaLocals};
 
-pub struct GVN;
+pub(super) struct GVN;
 
 impl<'tcx> crate::MirPass<'tcx> for GVN {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs
index 6cc7e0ee1e4..03ef808efec 100644
--- a/compiler/rustc_mir_transform/src/inline.rs
+++ b/compiler/rustc_mir_transform/src/inline.rs
@@ -32,9 +32,11 @@ pub(crate) mod cycle;
 
 const TOP_DOWN_DEPTH_LIMIT: usize = 5;
 
+// Made public so that `mir_drops_elaborated_and_const_checked` can be overridden
+// by custom rustc drivers, running all the steps by themselves. See #114628.
 pub struct Inline;
 
-#[derive(Copy, Clone, Debug)]
+#[derive(Clone, Debug)]
 struct CallSite<'tcx> {
     callee: Instance<'tcx>,
     fn_sig: ty::PolyFnSig<'tcx>,
@@ -156,7 +158,6 @@ impl<'tcx> Inliner<'tcx> {
             match self.try_inlining(caller_body, &callsite) {
                 Err(reason) => {
                     debug!("not-inlined {} [{}]", callsite.callee, reason);
-                    continue;
                 }
                 Ok(new_blocks) => {
                     debug!("inlined {}", callsite.callee);
@@ -638,7 +639,7 @@ impl<'tcx> Inliner<'tcx> {
             );
             let dest_ty = dest.ty(caller_body, self.tcx);
             let temp =
-                Place::from(self.new_call_temp(caller_body, &callsite, dest_ty, return_block));
+                Place::from(self.new_call_temp(caller_body, callsite, dest_ty, return_block));
             caller_body[callsite.block].statements.push(Statement {
                 source_info: callsite.source_info,
                 kind: StatementKind::Assign(Box::new((temp, dest))),
@@ -657,7 +658,7 @@ impl<'tcx> Inliner<'tcx> {
                 true,
                 self.new_call_temp(
                     caller_body,
-                    &callsite,
+                    callsite,
                     destination.ty(caller_body, self.tcx).ty,
                     return_block,
                 ),
@@ -665,7 +666,7 @@ impl<'tcx> Inliner<'tcx> {
         };
 
         // Copy the arguments if needed.
-        let args = self.make_call_args(args, &callsite, caller_body, &callee_body, return_block);
+        let args = self.make_call_args(args, callsite, caller_body, &callee_body, return_block);
 
         let mut integrator = Integrator {
             args: &args,
diff --git a/compiler/rustc_mir_transform/src/instsimplify.rs b/compiler/rustc_mir_transform/src/instsimplify.rs
index 4fbfa744e67..a9591bf3984 100644
--- a/compiler/rustc_mir_transform/src/instsimplify.rs
+++ b/compiler/rustc_mir_transform/src/instsimplify.rs
@@ -13,13 +13,13 @@ use rustc_target::spec::abi::Abi;
 use crate::simplify::simplify_duplicate_switch_targets;
 use crate::take_array;
 
-pub enum InstSimplify {
+pub(super) enum InstSimplify {
     BeforeInline,
     AfterSimplifyCfg,
 }
 
 impl InstSimplify {
-    pub fn name(&self) -> &'static str {
+    fn name(&self) -> &'static str {
         match self {
             InstSimplify::BeforeInline => "InstSimplify-before-inline",
             InstSimplify::AfterSimplifyCfg => "InstSimplify-after-simplifycfg",
diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs
index 02dd56e1b4f..e950c2cb72a 100644
--- a/compiler/rustc_mir_transform/src/jump_threading.rs
+++ b/compiler/rustc_mir_transform/src/jump_threading.rs
@@ -55,7 +55,7 @@ use tracing::{debug, instrument, trace};
 
 use crate::cost_checker::CostChecker;
 
-pub struct JumpThreading;
+pub(super) struct JumpThreading;
 
 const MAX_BACKTRACK: usize = 5;
 const MAX_COST: usize = 100;
diff --git a/compiler/rustc_mir_transform/src/known_panics_lint.rs b/compiler/rustc_mir_transform/src/known_panics_lint.rs
index 61405fb25c6..e964310c542 100644
--- a/compiler/rustc_mir_transform/src/known_panics_lint.rs
+++ b/compiler/rustc_mir_transform/src/known_panics_lint.rs
@@ -26,7 +26,7 @@ use tracing::{debug, instrument, trace};
 
 use crate::errors::{AssertLint, AssertLintKind};
 
-pub struct KnownPanicsLint;
+pub(super) struct KnownPanicsLint;
 
 impl<'tcx> crate::MirLint<'tcx> for KnownPanicsLint {
     fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) {
@@ -852,7 +852,7 @@ const MAX_ALLOC_LIMIT: u64 = 1024;
 
 /// The mode that `ConstProp` is allowed to run in for a given `Local`.
 #[derive(Clone, Copy, Debug, PartialEq)]
-pub enum ConstPropMode {
+enum ConstPropMode {
     /// The `Local` can be propagated into and reads of this `Local` can also be propagated.
     FullConstProp,
     /// The `Local` can only be propagated into and from its own block.
@@ -864,7 +864,7 @@ pub enum ConstPropMode {
 
 /// A visitor that determines locals in a MIR body
 /// that can be const propagated
-pub struct CanConstProp {
+struct CanConstProp {
     can_const_prop: IndexVec<Local, ConstPropMode>,
     // False at the beginning. Once set, no more assignments are allowed to that local.
     found_assignment: BitSet<Local>,
@@ -872,7 +872,7 @@ pub struct CanConstProp {
 
 impl CanConstProp {
     /// Returns true if `local` can be propagated
-    pub fn check<'tcx>(
+    fn check<'tcx>(
         tcx: TyCtxt<'tcx>,
         param_env: ParamEnv<'tcx>,
         body: &Body<'tcx>,
diff --git a/compiler/rustc_mir_transform/src/large_enums.rs b/compiler/rustc_mir_transform/src/large_enums.rs
index f02ba71ddc6..f61cf236f77 100644
--- a/compiler/rustc_mir_transform/src/large_enums.rs
+++ b/compiler/rustc_mir_transform/src/large_enums.rs
@@ -23,7 +23,7 @@ use rustc_target::abi::{HasDataLayout, Size, TagEncoding, Variants};
 /// In summary, what this does is at runtime determine which enum variant is active,
 /// and instead of copying all the bytes of the largest possible variant,
 /// copy only the bytes for the currently active variant.
-pub struct EnumSizeOpt {
+pub(super) struct EnumSizeOpt {
     pub(crate) discrepancy: u64,
 }
 
diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs
index 62e73ba2c8e..0bbbf047f63 100644
--- a/compiler/rustc_mir_transform/src/lib.rs
+++ b/compiler/rustc_mir_transform/src/lib.rs
@@ -11,6 +11,7 @@
 #![feature(round_char_boundary)]
 #![feature(try_blocks)]
 #![feature(yeet_expr)]
+#![warn(unreachable_pub)]
 // tidy-alphabetical-end
 
 use hir::ConstContext;
@@ -72,6 +73,8 @@ mod errors;
 mod ffi_unwind_calls;
 mod function_item_references;
 mod gvn;
+// Made public so that `mir_drops_elaborated_and_const_checked` can be overridden
+// by custom rustc drivers, running all the steps by themselves. See #114628.
 pub mod inline;
 mod instsimplify;
 mod jump_threading;
@@ -459,8 +462,8 @@ fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> &
     tcx.alloc_steal_mir(body)
 }
 
-// Made public such that `mir_drops_elaborated_and_const_checked` can be overridden
-// by custom rustc drivers, running all the steps by themselves.
+// Made public so that `mir_drops_elaborated_and_const_checked` can be overridden
+// by custom rustc drivers, running all the steps by themselves. See #114628.
 pub fn run_analysis_to_runtime_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
     assert!(body.phase == MirPhase::Analysis(AnalysisPhase::Initial));
     let did = body.source.def_id();
diff --git a/compiler/rustc_mir_transform/src/lint.rs b/compiler/rustc_mir_transform/src/lint.rs
index 746068064b8..23733994a8b 100644
--- a/compiler/rustc_mir_transform/src/lint.rs
+++ b/compiler/rustc_mir_transform/src/lint.rs
@@ -13,7 +13,7 @@ use rustc_mir_dataflow::impls::{MaybeStorageDead, MaybeStorageLive};
 use rustc_mir_dataflow::storage::always_storage_live_locals;
 use rustc_mir_dataflow::{Analysis, ResultsCursor};
 
-pub fn lint_body<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, when: String) {
+pub(super) fn lint_body<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, when: String) {
     let always_live_locals = &always_storage_live_locals(body);
 
     let maybe_storage_live = MaybeStorageLive::new(Cow::Borrowed(always_live_locals))
diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs
index 55eec332306..6d635606687 100644
--- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs
+++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs
@@ -7,7 +7,7 @@ use rustc_span::symbol::sym;
 
 use crate::take_array;
 
-pub struct LowerIntrinsics;
+pub(super) struct LowerIntrinsics;
 
 impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
@@ -35,20 +35,19 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
                         terminator.kind = TerminatorKind::Goto { target };
                     }
                     sym::forget => {
-                        if let Some(target) = *target {
-                            block.statements.push(Statement {
-                                source_info: terminator.source_info,
-                                kind: StatementKind::Assign(Box::new((
-                                    *destination,
-                                    Rvalue::Use(Operand::Constant(Box::new(ConstOperand {
-                                        span: terminator.source_info.span,
-                                        user_ty: None,
-                                        const_: Const::zero_sized(tcx.types.unit),
-                                    }))),
-                                ))),
-                            });
-                            terminator.kind = TerminatorKind::Goto { target };
-                        }
+                        let target = target.unwrap();
+                        block.statements.push(Statement {
+                            source_info: terminator.source_info,
+                            kind: StatementKind::Assign(Box::new((
+                                *destination,
+                                Rvalue::Use(Operand::Constant(Box::new(ConstOperand {
+                                    span: terminator.source_info.span,
+                                    user_ty: None,
+                                    const_: Const::zero_sized(tcx.types.unit),
+                                }))),
+                            ))),
+                        });
+                        terminator.kind = TerminatorKind::Goto { target };
                     }
                     sym::copy_nonoverlapping => {
                         let target = target.unwrap();
@@ -121,43 +120,41 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
                         terminator.kind = TerminatorKind::Goto { target };
                     }
                     sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
-                        if let Some(target) = *target {
-                            let Ok([lhs, rhs]) = take_array(args) else {
-                                bug!("Wrong arguments for {} intrinsic", intrinsic.name);
-                            };
-                            let bin_op = match intrinsic.name {
-                                sym::add_with_overflow => BinOp::AddWithOverflow,
-                                sym::sub_with_overflow => BinOp::SubWithOverflow,
-                                sym::mul_with_overflow => BinOp::MulWithOverflow,
-                                _ => bug!("unexpected intrinsic"),
-                            };
-                            block.statements.push(Statement {
-                                source_info: terminator.source_info,
-                                kind: StatementKind::Assign(Box::new((
-                                    *destination,
-                                    Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))),
-                                ))),
-                            });
-                            terminator.kind = TerminatorKind::Goto { target };
-                        }
+                        let target = target.unwrap();
+                        let Ok([lhs, rhs]) = take_array(args) else {
+                            bug!("Wrong arguments for {} intrinsic", intrinsic.name);
+                        };
+                        let bin_op = match intrinsic.name {
+                            sym::add_with_overflow => BinOp::AddWithOverflow,
+                            sym::sub_with_overflow => BinOp::SubWithOverflow,
+                            sym::mul_with_overflow => BinOp::MulWithOverflow,
+                            _ => bug!("unexpected intrinsic"),
+                        };
+                        block.statements.push(Statement {
+                            source_info: terminator.source_info,
+                            kind: StatementKind::Assign(Box::new((
+                                *destination,
+                                Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))),
+                            ))),
+                        });
+                        terminator.kind = TerminatorKind::Goto { target };
                     }
                     sym::size_of | sym::min_align_of => {
-                        if let Some(target) = *target {
-                            let tp_ty = generic_args.type_at(0);
-                            let null_op = match intrinsic.name {
-                                sym::size_of => NullOp::SizeOf,
-                                sym::min_align_of => NullOp::AlignOf,
-                                _ => bug!("unexpected intrinsic"),
-                            };
-                            block.statements.push(Statement {
-                                source_info: terminator.source_info,
-                                kind: StatementKind::Assign(Box::new((
-                                    *destination,
-                                    Rvalue::NullaryOp(null_op, tp_ty),
-                                ))),
-                            });
-                            terminator.kind = TerminatorKind::Goto { target };
-                        }
+                        let target = target.unwrap();
+                        let tp_ty = generic_args.type_at(0);
+                        let null_op = match intrinsic.name {
+                            sym::size_of => NullOp::SizeOf,
+                            sym::min_align_of => NullOp::AlignOf,
+                            _ => bug!("unexpected intrinsic"),
+                        };
+                        block.statements.push(Statement {
+                            source_info: terminator.source_info,
+                            kind: StatementKind::Assign(Box::new((
+                                *destination,
+                                Rvalue::NullaryOp(null_op, tp_ty),
+                            ))),
+                        });
+                        terminator.kind = TerminatorKind::Goto { target };
                     }
                     sym::read_via_copy => {
                         let Ok([arg]) = take_array(args) else {
@@ -219,17 +216,23 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
                         terminator.kind = TerminatorKind::Goto { target };
                     }
                     sym::discriminant_value => {
-                        if let (Some(target), Some(arg)) = (*target, args[0].node.place()) {
-                            let arg = tcx.mk_place_deref(arg);
-                            block.statements.push(Statement {
-                                source_info: terminator.source_info,
-                                kind: StatementKind::Assign(Box::new((
-                                    *destination,
-                                    Rvalue::Discriminant(arg),
-                                ))),
-                            });
-                            terminator.kind = TerminatorKind::Goto { target };
-                        }
+                        let target = target.unwrap();
+                        let Ok([arg]) = take_array(args) else {
+                            span_bug!(
+                                terminator.source_info.span,
+                                "Wrong arguments for discriminant_value intrinsic"
+                            );
+                        };
+                        let arg = arg.node.place().unwrap();
+                        let arg = tcx.mk_place_deref(arg);
+                        block.statements.push(Statement {
+                            source_info: terminator.source_info,
+                            kind: StatementKind::Assign(Box::new((
+                                *destination,
+                                Rvalue::Discriminant(arg),
+                            ))),
+                        });
+                        terminator.kind = TerminatorKind::Goto { target };
                     }
                     sym::offset => {
                         let target = target.unwrap();
@@ -267,7 +270,6 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
                                 Rvalue::Cast(CastKind::Transmute, arg.node, dst_ty),
                             ))),
                         });
-
                         if let Some(target) = *target {
                             terminator.kind = TerminatorKind::Goto { target };
                         } else {
@@ -299,7 +301,6 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
                                 Rvalue::Aggregate(Box::new(kind), fields.into()),
                             ))),
                         });
-
                         terminator.kind = TerminatorKind::Goto { target };
                     }
                     sym::ptr_metadata => {
diff --git a/compiler/rustc_mir_transform/src/lower_slice_len.rs b/compiler/rustc_mir_transform/src/lower_slice_len.rs
index 555309a7750..ca59d4d12ac 100644
--- a/compiler/rustc_mir_transform/src/lower_slice_len.rs
+++ b/compiler/rustc_mir_transform/src/lower_slice_len.rs
@@ -5,7 +5,7 @@ use rustc_hir::def_id::DefId;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
 
-pub struct LowerSliceLenCalls;
+pub(super) struct LowerSliceLenCalls;
 
 impl<'tcx> crate::MirPass<'tcx> for LowerSliceLenCalls {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
@@ -17,7 +17,7 @@ impl<'tcx> crate::MirPass<'tcx> for LowerSliceLenCalls {
     }
 }
 
-pub fn lower_slice_len_calls<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+fn lower_slice_len_calls<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
     let language_items = tcx.lang_items();
     let Some(slice_len_fn_item_def_id) = language_items.slice_len_fn() else {
         // there is no lang item to compare to :)
diff --git a/compiler/rustc_mir_transform/src/match_branches.rs b/compiler/rustc_mir_transform/src/match_branches.rs
index 233b39fb47a..2a2616b20a6 100644
--- a/compiler/rustc_mir_transform/src/match_branches.rs
+++ b/compiler/rustc_mir_transform/src/match_branches.rs
@@ -10,7 +10,7 @@ use rustc_type_ir::TyKind::*;
 
 use super::simplify::simplify_cfg;
 
-pub struct MatchBranchSimplification;
+pub(super) struct MatchBranchSimplification;
 
 impl<'tcx> crate::MirPass<'tcx> for MatchBranchSimplification {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
diff --git a/compiler/rustc_mir_transform/src/mentioned_items.rs b/compiler/rustc_mir_transform/src/mentioned_items.rs
index 41ce03caf08..7f9d0a5b698 100644
--- a/compiler/rustc_mir_transform/src/mentioned_items.rs
+++ b/compiler/rustc_mir_transform/src/mentioned_items.rs
@@ -5,7 +5,7 @@ use rustc_middle::ty::{self, TyCtxt};
 use rustc_session::Session;
 use rustc_span::source_map::Spanned;
 
-pub struct MentionedItems;
+pub(super) struct MentionedItems;
 
 struct MentionedItemsVisitor<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
diff --git a/compiler/rustc_mir_transform/src/multiple_return_terminators.rs b/compiler/rustc_mir_transform/src/multiple_return_terminators.rs
index 1b4972d487e..b6d6ef5de1d 100644
--- a/compiler/rustc_mir_transform/src/multiple_return_terminators.rs
+++ b/compiler/rustc_mir_transform/src/multiple_return_terminators.rs
@@ -7,7 +7,7 @@ use rustc_middle::ty::TyCtxt;
 
 use crate::simplify;
 
-pub struct MultipleReturnTerminators;
+pub(super) struct MultipleReturnTerminators;
 
 impl<'tcx> crate::MirPass<'tcx> for MultipleReturnTerminators {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
diff --git a/compiler/rustc_mir_transform/src/nrvo.rs b/compiler/rustc_mir_transform/src/nrvo.rs
index 94573a9d89b..98fa149e2bc 100644
--- a/compiler/rustc_mir_transform/src/nrvo.rs
+++ b/compiler/rustc_mir_transform/src/nrvo.rs
@@ -30,7 +30,7 @@ use tracing::{debug, trace};
 ///
 /// [#47954]: https://github.com/rust-lang/rust/pull/47954
 /// [#71003]: https://github.com/rust-lang/rust/pull/71003
-pub struct RenameReturnPlace;
+pub(super) struct RenameReturnPlace;
 
 impl<'tcx> crate::MirPass<'tcx> for RenameReturnPlace {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs
index 28d4e1a1c91..60ece5e7db9 100644
--- a/compiler/rustc_mir_transform/src/pass_manager.rs
+++ b/compiler/rustc_mir_transform/src/pass_manager.rs
@@ -269,12 +269,7 @@ pub(super) fn validate_body<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, when
     validate::Validator { when, mir_phase: body.phase }.run_pass(tcx, body);
 }
 
-pub(super) fn dump_mir_for_pass<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    body: &Body<'tcx>,
-    pass_name: &str,
-    is_after: bool,
-) {
+fn dump_mir_for_pass<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, pass_name: &str, is_after: bool) {
     mir::dump_mir(
         tcx,
         true,
diff --git a/compiler/rustc_mir_transform/src/prettify.rs b/compiler/rustc_mir_transform/src/prettify.rs
index ad71c622660..ef011d230c2 100644
--- a/compiler/rustc_mir_transform/src/prettify.rs
+++ b/compiler/rustc_mir_transform/src/prettify.rs
@@ -15,7 +15,7 @@ use rustc_session::Session;
 ///
 /// Thus after this pass, all the successors of a block are later than it in the
 /// `IndexVec`, unless that successor is a back-edge (such as from a loop).
-pub struct ReorderBasicBlocks;
+pub(super) struct ReorderBasicBlocks;
 
 impl<'tcx> crate::MirPass<'tcx> for ReorderBasicBlocks {
     fn is_enabled(&self, _session: &Session) -> bool {
@@ -43,7 +43,7 @@ impl<'tcx> crate::MirPass<'tcx> for ReorderBasicBlocks {
 /// assigned or referenced will have a smaller number.
 ///
 /// (Does not reorder arguments nor the [`RETURN_PLACE`].)
-pub struct ReorderLocals;
+pub(super) struct ReorderLocals;
 
 impl<'tcx> crate::MirPass<'tcx> for ReorderLocals {
     fn is_enabled(&self, _session: &Session) -> bool {
@@ -135,8 +135,8 @@ impl<'tcx> Visitor<'tcx> for LocalFinder {
 }
 
 struct LocalUpdater<'tcx> {
-    pub map: IndexVec<Local, Local>,
-    pub tcx: TyCtxt<'tcx>,
+    map: IndexVec<Local, Local>,
+    tcx: TyCtxt<'tcx>,
 }
 
 impl<'tcx> MutVisitor<'tcx> for LocalUpdater<'tcx> {
diff --git a/compiler/rustc_mir_transform/src/promote_consts.rs b/compiler/rustc_mir_transform/src/promote_consts.rs
index cf0a569ffa4..65309f63d59 100644
--- a/compiler/rustc_mir_transform/src/promote_consts.rs
+++ b/compiler/rustc_mir_transform/src/promote_consts.rs
@@ -37,7 +37,7 @@ use tracing::{debug, instrument};
 /// After this pass is run, `promoted_fragments` will hold the MIR body corresponding to each
 /// newly created `Constant`.
 #[derive(Default)]
-pub struct PromoteTemps<'tcx> {
+pub(super) struct PromoteTemps<'tcx> {
     pub promoted_fragments: Cell<IndexVec<Promoted, Body<'tcx>>>,
 }
 
diff --git a/compiler/rustc_mir_transform/src/ref_prop.rs b/compiler/rustc_mir_transform/src/ref_prop.rs
index 4a447d24cce..25b98786c66 100644
--- a/compiler/rustc_mir_transform/src/ref_prop.rs
+++ b/compiler/rustc_mir_transform/src/ref_prop.rs
@@ -70,7 +70,7 @@ use crate::ssa::{SsaLocals, StorageLiveLocals};
 ///
 /// For immutable borrows, we do not need to preserve such uniqueness property,
 /// so we perform all the possible instantiations without removing the `_1 = &_2` statement.
-pub struct ReferencePropagation;
+pub(super) struct ReferencePropagation;
 
 impl<'tcx> crate::MirPass<'tcx> for ReferencePropagation {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
diff --git a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs
index ccba8d015e3..37197c3f573 100644
--- a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs
+++ b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs
@@ -8,7 +8,7 @@ use tracing::debug;
 /// A pass that removes noop landing pads and replaces jumps to them with
 /// `UnwindAction::Continue`. This is important because otherwise LLVM generates
 /// terrible code for these.
-pub struct RemoveNoopLandingPads;
+pub(super) struct RemoveNoopLandingPads;
 
 impl<'tcx> crate::MirPass<'tcx> for RemoveNoopLandingPads {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
diff --git a/compiler/rustc_mir_transform/src/remove_place_mention.rs b/compiler/rustc_mir_transform/src/remove_place_mention.rs
index 5801fdedceb..71399eb72f0 100644
--- a/compiler/rustc_mir_transform/src/remove_place_mention.rs
+++ b/compiler/rustc_mir_transform/src/remove_place_mention.rs
@@ -4,7 +4,7 @@ use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
 use tracing::trace;
 
-pub struct RemovePlaceMention;
+pub(super) struct RemovePlaceMention;
 
 impl<'tcx> crate::MirPass<'tcx> for RemovePlaceMention {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
diff --git a/compiler/rustc_mir_transform/src/remove_storage_markers.rs b/compiler/rustc_mir_transform/src/remove_storage_markers.rs
index 329b30d3890..3ecb4a8994f 100644
--- a/compiler/rustc_mir_transform/src/remove_storage_markers.rs
+++ b/compiler/rustc_mir_transform/src/remove_storage_markers.rs
@@ -4,7 +4,7 @@ use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
 use tracing::trace;
 
-pub struct RemoveStorageMarkers;
+pub(super) struct RemoveStorageMarkers;
 
 impl<'tcx> crate::MirPass<'tcx> for RemoveStorageMarkers {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
diff --git a/compiler/rustc_mir_transform/src/remove_uninit_drops.rs b/compiler/rustc_mir_transform/src/remove_uninit_drops.rs
index aafe971311d..c58f492655a 100644
--- a/compiler/rustc_mir_transform/src/remove_uninit_drops.rs
+++ b/compiler/rustc_mir_transform/src/remove_uninit_drops.rs
@@ -14,7 +14,7 @@ use rustc_target::abi::FieldIdx;
 /// like [#90770].
 ///
 /// [#90770]: https://github.com/rust-lang/rust/issues/90770
-pub struct RemoveUninitDrops;
+pub(super) struct RemoveUninitDrops;
 
 impl<'tcx> crate::MirPass<'tcx> for RemoveUninitDrops {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
diff --git a/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs b/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs
index 43109aae0fb..28925ba1beb 100644
--- a/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs
+++ b/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs
@@ -10,7 +10,7 @@ use tracing::{debug, trace};
 
 use super::simplify::simplify_cfg;
 
-pub struct RemoveUnneededDrops;
+pub(super) struct RemoveUnneededDrops;
 
 impl<'tcx> crate::MirPass<'tcx> for RemoveUnneededDrops {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
diff --git a/compiler/rustc_mir_transform/src/remove_zsts.rs b/compiler/rustc_mir_transform/src/remove_zsts.rs
index 9aa46bd4fba..f13bb1c5993 100644
--- a/compiler/rustc_mir_transform/src/remove_zsts.rs
+++ b/compiler/rustc_mir_transform/src/remove_zsts.rs
@@ -4,7 +4,7 @@ use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 
-pub struct RemoveZsts;
+pub(super) struct RemoveZsts;
 
 impl<'tcx> crate::MirPass<'tcx> for RemoveZsts {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
diff --git a/compiler/rustc_mir_transform/src/required_consts.rs b/compiler/rustc_mir_transform/src/required_consts.rs
index 50637e2ac03..ebcf5b5d27b 100644
--- a/compiler/rustc_mir_transform/src/required_consts.rs
+++ b/compiler/rustc_mir_transform/src/required_consts.rs
@@ -1,7 +1,7 @@
 use rustc_middle::mir::visit::Visitor;
 use rustc_middle::mir::{traversal, Body, ConstOperand, Location};
 
-pub struct RequiredConstsVisitor<'a, 'tcx> {
+pub(super) struct RequiredConstsVisitor<'a, 'tcx> {
     required_consts: &'a mut Vec<ConstOperand<'tcx>>,
 }
 
@@ -10,7 +10,7 @@ impl<'a, 'tcx> RequiredConstsVisitor<'a, 'tcx> {
         RequiredConstsVisitor { required_consts }
     }
 
-    pub fn compute_required_consts(body: &mut Body<'tcx>) {
+    pub(super) fn compute_required_consts(body: &mut Body<'tcx>) {
         let mut required_consts = Vec::new();
         let mut required_consts_visitor = RequiredConstsVisitor::new(&mut required_consts);
         for (bb, bb_data) in traversal::reverse_postorder(&body) {
diff --git a/compiler/rustc_mir_transform/src/reveal_all.rs b/compiler/rustc_mir_transform/src/reveal_all.rs
index 29312a99cbc..3db962bd94a 100644
--- a/compiler/rustc_mir_transform/src/reveal_all.rs
+++ b/compiler/rustc_mir_transform/src/reveal_all.rs
@@ -4,7 +4,7 @@ use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 
-pub struct RevealAll;
+pub(super) struct RevealAll;
 
 impl<'tcx> crate::MirPass<'tcx> for RevealAll {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs
index 8c70e429118..cf8ef580b27 100644
--- a/compiler/rustc_mir_transform/src/shim.rs
+++ b/compiler/rustc_mir_transform/src/shim.rs
@@ -26,7 +26,7 @@ use crate::{
 
 mod async_destructor_ctor;
 
-pub fn provide(providers: &mut Providers) {
+pub(super) fn provide(providers: &mut Providers) {
     providers.mir_shims = make_shim;
 }
 
@@ -331,7 +331,7 @@ fn new_body<'tcx>(
     body
 }
 
-pub struct DropShimElaborator<'a, 'tcx> {
+pub(super) struct DropShimElaborator<'a, 'tcx> {
     pub body: &'a Body<'tcx>,
     pub patch: MirPatch<'tcx>,
     pub tcx: TyCtxt<'tcx>,
@@ -913,7 +913,7 @@ fn build_call_shim<'tcx>(
     body
 }
 
-pub fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> Body<'_> {
+pub(super) fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> Body<'_> {
     debug_assert!(tcx.is_constructor(ctor_id));
 
     let param_env = tcx.param_env_reveal_all_normalized(ctor_id);
diff --git a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs
index 50810f23113..c88953a1d1a 100644
--- a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs
+++ b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs
@@ -23,7 +23,7 @@ use tracing::debug;
 
 use super::{local_decls_for_sig, new_body};
 
-pub fn build_async_destructor_ctor_shim<'tcx>(
+pub(super) fn build_async_destructor_ctor_shim<'tcx>(
     tcx: TyCtxt<'tcx>,
     def_id: DefId,
     ty: Option<Ty<'tcx>>,
diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs
index 1478b86d3c7..cb8e1dfda98 100644
--- a/compiler/rustc_mir_transform/src/simplify.rs
+++ b/compiler/rustc_mir_transform/src/simplify.rs
@@ -35,7 +35,7 @@ use rustc_span::DUMMY_SP;
 use smallvec::SmallVec;
 use tracing::{debug, trace};
 
-pub enum SimplifyCfg {
+pub(super) enum SimplifyCfg {
     Initial,
     PromoteConsts,
     RemoveFalseEdges,
@@ -50,7 +50,7 @@ pub enum SimplifyCfg {
 }
 
 impl SimplifyCfg {
-    pub fn name(&self) -> &'static str {
+    fn name(&self) -> &'static str {
         match self {
             SimplifyCfg::Initial => "SimplifyCfg-initial",
             SimplifyCfg::PromoteConsts => "SimplifyCfg-promote-consts",
@@ -66,7 +66,7 @@ impl SimplifyCfg {
     }
 }
 
-pub(crate) fn simplify_cfg(body: &mut Body<'_>) {
+pub(super) fn simplify_cfg(body: &mut Body<'_>) {
     CfgSimplifier::new(body).simplify();
     remove_dead_blocks(body);
 
@@ -85,13 +85,13 @@ impl<'tcx> crate::MirPass<'tcx> for SimplifyCfg {
     }
 }
 
-pub struct CfgSimplifier<'a, 'tcx> {
+struct CfgSimplifier<'a, 'tcx> {
     basic_blocks: &'a mut IndexSlice<BasicBlock, BasicBlockData<'tcx>>,
     pred_count: IndexVec<BasicBlock, u32>,
 }
 
 impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> {
-    pub fn new(body: &'a mut Body<'tcx>) -> Self {
+    fn new(body: &'a mut Body<'tcx>) -> Self {
         let mut pred_count = IndexVec::from_elem(0u32, &body.basic_blocks);
 
         // we can't use mir.predecessors() here because that counts
@@ -111,7 +111,7 @@ impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> {
         CfgSimplifier { basic_blocks, pred_count }
     }
 
-    pub fn simplify(mut self) {
+    fn simplify(mut self) {
         self.strip_nops();
 
         // Vec of the blocks that should be merged. We store the indices here, instead of the
@@ -280,7 +280,7 @@ impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> {
     }
 }
 
-pub fn simplify_duplicate_switch_targets(terminator: &mut Terminator<'_>) {
+pub(super) fn simplify_duplicate_switch_targets(terminator: &mut Terminator<'_>) {
     if let TerminatorKind::SwitchInt { targets, .. } = &mut terminator.kind {
         let otherwise = targets.otherwise();
         if targets.iter().any(|t| t.1 == otherwise) {
@@ -292,7 +292,7 @@ pub fn simplify_duplicate_switch_targets(terminator: &mut Terminator<'_>) {
     }
 }
 
-pub(crate) fn remove_dead_blocks(body: &mut Body<'_>) {
+pub(super) fn remove_dead_blocks(body: &mut Body<'_>) {
     let should_deduplicate_unreachable = |bbdata: &BasicBlockData<'_>| {
         // CfgSimplifier::simplify leaves behind some unreachable basic blocks without a
         // terminator. Those blocks will be deleted by remove_dead_blocks, but we run just
@@ -360,7 +360,7 @@ pub(crate) fn remove_dead_blocks(body: &mut Body<'_>) {
     }
 }
 
-pub enum SimplifyLocals {
+pub(super) enum SimplifyLocals {
     BeforeConstProp,
     AfterGVN,
     Final,
@@ -385,7 +385,7 @@ impl<'tcx> crate::MirPass<'tcx> for SimplifyLocals {
     }
 }
 
-pub fn remove_unused_definitions<'tcx>(body: &mut Body<'tcx>) {
+pub(super) fn remove_unused_definitions<'tcx>(body: &mut Body<'tcx>) {
     // First, we're going to get a count of *actual* uses for every `Local`.
     let mut used_locals = UsedLocals::new(body);
 
@@ -397,7 +397,7 @@ pub fn remove_unused_definitions<'tcx>(body: &mut Body<'tcx>) {
     remove_unused_definitions_helper(&mut used_locals, body);
 }
 
-pub fn simplify_locals<'tcx>(body: &mut Body<'tcx>, tcx: TyCtxt<'tcx>) {
+fn simplify_locals<'tcx>(body: &mut Body<'tcx>, tcx: TyCtxt<'tcx>) {
     // First, we're going to get a count of *actual* uses for every `Local`.
     let mut used_locals = UsedLocals::new(body);
 
diff --git a/compiler/rustc_mir_transform/src/simplify_branches.rs b/compiler/rustc_mir_transform/src/simplify_branches.rs
index 5a014bb7346..e83b4727c48 100644
--- a/compiler/rustc_mir_transform/src/simplify_branches.rs
+++ b/compiler/rustc_mir_transform/src/simplify_branches.rs
@@ -2,10 +2,11 @@ use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
 use tracing::trace;
 
-pub enum SimplifyConstCondition {
+pub(super) enum SimplifyConstCondition {
     AfterConstProp,
     Final,
 }
+
 /// A pass that replaces a branch with a goto when its condition is known.
 impl<'tcx> crate::MirPass<'tcx> for SimplifyConstCondition {
     fn name(&self) -> &'static str {
diff --git a/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs b/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs
index bd30ecc59b3..644bcb58d56 100644
--- a/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs
+++ b/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs
@@ -23,7 +23,7 @@ use tracing::trace;
 /// ```ignore (MIR)
 /// switchInt(_4) -> [43i32: bb3, otherwise: bb2];
 /// ```
-pub struct SimplifyComparisonIntegral;
+pub(super) struct SimplifyComparisonIntegral;
 
 impl<'tcx> crate::MirPass<'tcx> for SimplifyComparisonIntegral {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
diff --git a/compiler/rustc_mir_transform/src/single_use_consts.rs b/compiler/rustc_mir_transform/src/single_use_consts.rs
index 64a92872830..26059268c37 100644
--- a/compiler/rustc_mir_transform/src/single_use_consts.rs
+++ b/compiler/rustc_mir_transform/src/single_use_consts.rs
@@ -19,7 +19,7 @@ use rustc_middle::ty::TyCtxt;
 ///
 /// It also removes *never*-used constants, since it had all the information
 /// needed to do that too, including updating the debug info.
-pub struct SingleUseConsts;
+pub(super) struct SingleUseConsts;
 
 impl<'tcx> crate::MirPass<'tcx> for SingleUseConsts {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
diff --git a/compiler/rustc_mir_transform/src/sroa.rs b/compiler/rustc_mir_transform/src/sroa.rs
index 3c5ccc0c99a..e6dd2cfd862 100644
--- a/compiler/rustc_mir_transform/src/sroa.rs
+++ b/compiler/rustc_mir_transform/src/sroa.rs
@@ -11,7 +11,7 @@ use rustc_mir_dataflow::value_analysis::{excluded_locals, iter_fields};
 use rustc_target::abi::{FieldIdx, FIRST_VARIANT};
 use tracing::{debug, instrument};
 
-pub struct ScalarReplacementOfAggregates;
+pub(super) struct ScalarReplacementOfAggregates;
 
 impl<'tcx> crate::MirPass<'tcx> for ScalarReplacementOfAggregates {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
diff --git a/compiler/rustc_mir_transform/src/ssa.rs b/compiler/rustc_mir_transform/src/ssa.rs
index c1254766f0a..cf8622cadd1 100644
--- a/compiler/rustc_mir_transform/src/ssa.rs
+++ b/compiler/rustc_mir_transform/src/ssa.rs
@@ -16,7 +16,7 @@ use rustc_middle::mir::*;
 use rustc_middle::ty::{ParamEnv, TyCtxt};
 use tracing::{debug, instrument, trace};
 
-pub struct SsaLocals {
+pub(super) struct SsaLocals {
     /// Assignments to each local. This defines whether the local is SSA.
     assignments: IndexVec<Local, Set1<DefLocation>>,
     /// We visit the body in reverse postorder, to ensure each local is assigned before it is used.
@@ -32,14 +32,18 @@ pub struct SsaLocals {
     borrowed_locals: BitSet<Local>,
 }
 
-pub enum AssignedValue<'a, 'tcx> {
+pub(super) enum AssignedValue<'a, 'tcx> {
     Arg,
     Rvalue(&'a mut Rvalue<'tcx>),
     Terminator,
 }
 
 impl SsaLocals {
-    pub fn new<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, param_env: ParamEnv<'tcx>) -> SsaLocals {
+    pub(super) fn new<'tcx>(
+        tcx: TyCtxt<'tcx>,
+        body: &Body<'tcx>,
+        param_env: ParamEnv<'tcx>,
+    ) -> SsaLocals {
         let assignment_order = Vec::with_capacity(body.local_decls.len());
 
         let assignments = IndexVec::from_elem(Set1::Empty, &body.local_decls);
@@ -101,25 +105,25 @@ impl SsaLocals {
         ssa
     }
 
-    pub fn num_locals(&self) -> usize {
+    pub(super) fn num_locals(&self) -> usize {
         self.assignments.len()
     }
 
-    pub fn locals(&self) -> impl Iterator<Item = Local> {
+    pub(super) fn locals(&self) -> impl Iterator<Item = Local> {
         self.assignments.indices()
     }
 
-    pub fn is_ssa(&self, local: Local) -> bool {
+    pub(super) fn is_ssa(&self, local: Local) -> bool {
         matches!(self.assignments[local], Set1::One(_))
     }
 
     /// Return the number of uses if a local that are not "Deref".
-    pub fn num_direct_uses(&self, local: Local) -> u32 {
+    pub(super) fn num_direct_uses(&self, local: Local) -> u32 {
         self.direct_uses[local]
     }
 
     #[inline]
-    pub fn assignment_dominates(
+    pub(super) fn assignment_dominates(
         &self,
         dominators: &Dominators<BasicBlock>,
         local: Local,
@@ -131,7 +135,7 @@ impl SsaLocals {
         }
     }
 
-    pub fn assignments<'a, 'tcx>(
+    pub(super) fn assignments<'a, 'tcx>(
         &'a self,
         body: &'a Body<'tcx>,
     ) -> impl Iterator<Item = (Local, &'a Rvalue<'tcx>, Location)> + 'a {
@@ -148,7 +152,7 @@ impl SsaLocals {
         })
     }
 
-    pub fn for_each_assignment_mut<'tcx>(
+    pub(super) fn for_each_assignment_mut<'tcx>(
         &self,
         basic_blocks: &mut IndexSlice<BasicBlock, BasicBlockData<'tcx>>,
         mut f: impl FnMut(Local, AssignedValue<'_, 'tcx>, Location),
@@ -194,17 +198,17 @@ impl SsaLocals {
     ///   _d => _a // transitively through _c
     ///
     /// Exception: we do not see through the return place, as it cannot be instantiated.
-    pub fn copy_classes(&self) -> &IndexSlice<Local, Local> {
+    pub(super) fn copy_classes(&self) -> &IndexSlice<Local, Local> {
         &self.copy_classes
     }
 
     /// Set of SSA locals that are immutably borrowed.
-    pub fn borrowed_locals(&self) -> &BitSet<Local> {
+    pub(super) fn borrowed_locals(&self) -> &BitSet<Local> {
         &self.borrowed_locals
     }
 
     /// Make a property uniform on a copy equivalence class by removing elements.
-    pub fn meet_copy_equivalence(&self, property: &mut BitSet<Local>) {
+    pub(super) fn meet_copy_equivalence(&self, property: &mut BitSet<Local>) {
         // Consolidate to have a local iff all its copies are.
         //
         // `copy_classes` defines equivalence classes between locals. The `local`s that recursively
diff --git a/compiler/rustc_mir_transform/src/unreachable_enum_branching.rs b/compiler/rustc_mir_transform/src/unreachable_enum_branching.rs
index 51a322628ee..6957394ed10 100644
--- a/compiler/rustc_mir_transform/src/unreachable_enum_branching.rs
+++ b/compiler/rustc_mir_transform/src/unreachable_enum_branching.rs
@@ -12,7 +12,7 @@ use rustc_middle::ty::{Ty, TyCtxt};
 use rustc_target::abi::{Abi, Variants};
 use tracing::trace;
 
-pub struct UnreachableEnumBranching;
+pub(super) struct UnreachableEnumBranching;
 
 fn get_discriminant_local(terminator: &TerminatorKind<'_>) -> Option<Local> {
     if let TerminatorKind::SwitchInt { discr: Operand::Move(p), .. } = terminator {
diff --git a/compiler/rustc_mir_transform/src/unreachable_prop.rs b/compiler/rustc_mir_transform/src/unreachable_prop.rs
index b8da86f1a8d..c60cbae2142 100644
--- a/compiler/rustc_mir_transform/src/unreachable_prop.rs
+++ b/compiler/rustc_mir_transform/src/unreachable_prop.rs
@@ -10,7 +10,7 @@ use rustc_middle::mir::*;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_target::abi::Size;
 
-pub struct UnreachablePropagation;
+pub(super) struct UnreachablePropagation;
 
 impl crate::MirPass<'_> for UnreachablePropagation {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs
index 69e2592e82c..18865c73f75 100644
--- a/compiler/rustc_mir_transform/src/validate.rs
+++ b/compiler/rustc_mir_transform/src/validate.rs
@@ -1,6 +1,7 @@
 //! Validates the MIR to ensure that invariants are upheld.
 
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_hir as hir;
 use rustc_hir::LangItem;
 use rustc_index::bit_set::BitSet;
 use rustc_index::IndexVec;
@@ -25,7 +26,7 @@ enum EdgeKind {
     Normal,
 }
 
-pub struct Validator {
+pub(super) struct Validator {
     /// Describes at which point in the pipeline this validation is happening.
     pub when: String,
     /// The phase for which we are upholding the dialect. If the given phase forbids a specific
@@ -530,7 +531,7 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> {
 ///
 /// `caller_body` is used to detect cycles in MIR inlining and MIR validation before
 /// `optimized_mir` is available.
-pub fn validate_types<'tcx>(
+pub(super) fn validate_types<'tcx>(
     tcx: TyCtxt<'tcx>,
     mir_phase: MirPhase,
     param_env: ty::ParamEnv<'tcx>,
@@ -714,7 +715,17 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                             // since we may be in the process of computing this MIR in the
                             // first place.
                             let layout = if def_id == self.caller_body.source.def_id() {
-                                // FIXME: This is not right for async closures.
+                                self.caller_body.coroutine_layout_raw()
+                            } else if let Some(hir::CoroutineKind::Desugared(
+                                _,
+                                hir::CoroutineSource::Closure,
+                            )) = self.tcx.coroutine_kind(def_id)
+                                && let ty::ClosureKind::FnOnce =
+                                    args.as_coroutine().kind_ty().to_opt_closure_kind().unwrap()
+                                && self.caller_body.source.def_id()
+                                    == self.tcx.coroutine_by_move_body_def_id(def_id)
+                            {
+                                // Same if this is the by-move body of a coroutine-closure.
                                 self.caller_body.coroutine_layout_raw()
                             } else {
                                 self.tcx.coroutine_layout(def_id, args.as_coroutine().kind_ty())
diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs
index 8515ab45de2..093697a290c 100644
--- a/compiler/rustc_monomorphize/src/collector.rs
+++ b/compiler/rustc_monomorphize/src/collector.rs
@@ -1041,8 +1041,11 @@ fn find_vtable_types_for_unsizing<'tcx>(
     match (source_ty.kind(), target_ty.kind()) {
         (&ty::Ref(_, a, _), &ty::Ref(_, b, _) | &ty::RawPtr(b, _))
         | (&ty::RawPtr(a, _), &ty::RawPtr(b, _)) => ptr_vtable(a, b),
-        (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) if def_a.is_box() && def_b.is_box() => {
-            ptr_vtable(source_ty.boxed_ty(), target_ty.boxed_ty())
+        (_, _)
+            if let Some(source_boxed) = source_ty.boxed_ty()
+                && let Some(target_boxed) = target_ty.boxed_ty() =>
+        {
+            ptr_vtable(source_boxed, target_boxed)
         }
 
         // T as dyn* Trait
diff --git a/compiler/rustc_monomorphize/src/lib.rs b/compiler/rustc_monomorphize/src/lib.rs
index b22e8e30465..91101ddd590 100644
--- a/compiler/rustc_monomorphize/src/lib.rs
+++ b/compiler/rustc_monomorphize/src/lib.rs
@@ -1,5 +1,7 @@
 // tidy-alphabetical-start
 #![feature(array_windows)]
+#![feature(if_let_guard)]
+#![feature(let_chains)]
 #![warn(unreachable_pub)]
 // tidy-alphabetical-end
 
diff --git a/compiler/rustc_next_trait_solver/src/solve/normalizes_to/mod.rs b/compiler/rustc_next_trait_solver/src/solve/normalizes_to/mod.rs
index c7a9846cd97..17b6ec7e2bb 100644
--- a/compiler/rustc_next_trait_solver/src/solve/normalizes_to/mod.rs
+++ b/compiler/rustc_next_trait_solver/src/solve/normalizes_to/mod.rs
@@ -3,7 +3,7 @@ mod inherent;
 mod opaque_types;
 mod weak_types;
 
-use rustc_type_ir::fast_reject::{DeepRejectCtxt, TreatParams};
+use rustc_type_ir::fast_reject::DeepRejectCtxt;
 use rustc_type_ir::inherent::*;
 use rustc_type_ir::lang_items::TraitSolverLangItem;
 use rustc_type_ir::{self as ty, Interner, NormalizesTo, Upcast as _};
@@ -106,6 +106,12 @@ where
         if let Some(projection_pred) = assumption.as_projection_clause() {
             if projection_pred.projection_def_id() == goal.predicate.def_id() {
                 let cx = ecx.cx();
+                if !DeepRejectCtxt::relate_rigid_rigid(ecx.cx()).args_may_unify(
+                    goal.predicate.alias.args,
+                    projection_pred.skip_binder().projection_term.args,
+                ) {
+                    return Err(NoSolution);
+                }
                 ecx.probe_trait_candidate(source).enter(|ecx| {
                     let assumption_projection_pred =
                         ecx.instantiate_binder_with_infer(projection_pred);
@@ -144,7 +150,7 @@ where
 
         let goal_trait_ref = goal.predicate.alias.trait_ref(cx);
         let impl_trait_ref = cx.impl_trait_ref(impl_def_id);
-        if !DeepRejectCtxt::new(ecx.cx(), TreatParams::ForLookup).args_may_unify(
+        if !DeepRejectCtxt::relate_rigid_infer(ecx.cx()).args_may_unify(
             goal.predicate.alias.trait_ref(cx).args,
             impl_trait_ref.skip_binder().args,
         ) {
diff --git a/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs b/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs
index 67b001d0cce..683d8dab3b2 100644
--- a/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs
+++ b/compiler/rustc_next_trait_solver/src/solve/trait_goals.rs
@@ -2,7 +2,7 @@
 
 use rustc_ast_ir::Movability;
 use rustc_type_ir::data_structures::IndexSet;
-use rustc_type_ir::fast_reject::{DeepRejectCtxt, TreatParams};
+use rustc_type_ir::fast_reject::DeepRejectCtxt;
 use rustc_type_ir::inherent::*;
 use rustc_type_ir::lang_items::TraitSolverLangItem;
 use rustc_type_ir::visit::TypeVisitableExt as _;
@@ -47,7 +47,7 @@ where
         let cx = ecx.cx();
 
         let impl_trait_ref = cx.impl_trait_ref(impl_def_id);
-        if !DeepRejectCtxt::new(ecx.cx(), TreatParams::ForLookup)
+        if !DeepRejectCtxt::relate_rigid_infer(ecx.cx())
             .args_may_unify(goal.predicate.trait_ref.args, impl_trait_ref.skip_binder().args)
         {
             return Err(NoSolution);
@@ -124,6 +124,13 @@ where
             if trait_clause.def_id() == goal.predicate.def_id()
                 && trait_clause.polarity() == goal.predicate.polarity
             {
+                if !DeepRejectCtxt::relate_rigid_rigid(ecx.cx()).args_may_unify(
+                    goal.predicate.trait_ref.args,
+                    trait_clause.skip_binder().trait_ref.args,
+                ) {
+                    return Err(NoSolution);
+                }
+
                 ecx.probe_trait_candidate(source).enter(|ecx| {
                     let assumption_trait_pred = ecx.instantiate_binder_with_infer(trait_clause);
                     ecx.eq(
diff --git a/compiler/rustc_parse/Cargo.toml b/compiler/rustc_parse/Cargo.toml
index b33896154f2..c59ae48a07d 100644
--- a/compiler/rustc_parse/Cargo.toml
+++ b/compiler/rustc_parse/Cargo.toml
@@ -12,6 +12,7 @@ rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_feature = { path = "../rustc_feature" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
+rustc_index = { path = "../rustc_index" }
 rustc_lexer = { path = "../rustc_lexer" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_session = { path = "../rustc_session" }
diff --git a/compiler/rustc_parse/messages.ftl b/compiler/rustc_parse/messages.ftl
index 8e8d91ce4d0..ec9a676ea31 100644
--- a/compiler/rustc_parse/messages.ftl
+++ b/compiler/rustc_parse/messages.ftl
@@ -381,6 +381,7 @@ parse_invalid_char_in_escape_msg = invalid character in {$is_hex ->
     *[false] unicode
     } escape
 
+
 parse_invalid_comparison_operator = invalid comparison operator `{$invalid}`
     .use_instead = `{$invalid}` is not a valid comparison operator, use `{$correct}`
     .spaceship_operator_invalid = `<=>` is not a valid comparison operator, use `std::cmp::Ordering`
@@ -581,6 +582,11 @@ parse_missing_trait_in_trait_impl = missing trait in a trait impl
     .suggestion_add_trait = add a trait here
     .suggestion_remove_for = for an inherent impl, drop this `for`
 
+parse_misspelled_kw = {$is_incorrect_case ->
+                    [true] write keyword `{$similar_kw}` in lowercase
+                    *[false] there is a keyword `{$similar_kw}` with a similar name
+}
+
 parse_modifier_lifetime = `{$modifier}` may only modify trait bounds, not lifetime bounds
     .suggestion = remove the `{$modifier}`
 
diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs
index f30939093c2..b7232ff21ca 100644
--- a/compiler/rustc_parse/src/lexer/mod.rs
+++ b/compiler/rustc_parse/src/lexer/mod.rs
@@ -13,7 +13,6 @@ use rustc_session::lint::builtin::{
 };
 use rustc_session::lint::BuiltinLintDiag;
 use rustc_session::parse::ParseSess;
-use rustc_span::edition::Edition;
 use rustc_span::symbol::Symbol;
 use rustc_span::{BytePos, Pos, Span};
 use tracing::debug;
@@ -188,9 +187,7 @@ impl<'psess, 'src> StringReader<'psess, 'src> {
                     preceded_by_whitespace = true;
                     continue;
                 }
-                rustc_lexer::TokenKind::Ident => {
-                    self.ident(start)
-                }
+                rustc_lexer::TokenKind::Ident => self.ident(start),
                 rustc_lexer::TokenKind::RawIdent => {
                     let sym = nfc_normalize(self.str_from(start + BytePos(2)));
                     let span = self.mk_sp(start, self.pos);
@@ -205,20 +202,31 @@ impl<'psess, 'src> StringReader<'psess, 'src> {
                     self.report_unknown_prefix(start);
                     self.ident(start)
                 }
-                rustc_lexer::TokenKind::InvalidIdent
-                | rustc_lexer::TokenKind::InvalidPrefix
+                rustc_lexer::TokenKind::UnknownPrefixLifetime => {
+                    self.report_unknown_prefix(start);
+                    // Include the leading `'` in the real identifier, for macro
+                    // expansion purposes. See #12512 for the gory details of why
+                    // this is necessary.
+                    let lifetime_name = self.str_from(start);
+                    self.last_lifetime = Some(self.mk_sp(start, start + BytePos(1)));
+                    let ident = Symbol::intern(lifetime_name);
+                    token::Lifetime(ident, IdentIsRaw::No)
+                }
+                rustc_lexer::TokenKind::InvalidIdent | rustc_lexer::TokenKind::InvalidPrefix
                     // Do not recover an identifier with emoji if the codepoint is a confusable
                     // with a recoverable substitution token, like `➖`.
-                    if !UNICODE_ARRAY
-                        .iter()
-                        .any(|&(c, _, _)| {
-                            let sym = self.str_from(start);
-                            sym.chars().count() == 1 && c == sym.chars().next().unwrap()
-                        }) =>
+                    if !UNICODE_ARRAY.iter().any(|&(c, _, _)| {
+                        let sym = self.str_from(start);
+                        sym.chars().count() == 1 && c == sym.chars().next().unwrap()
+                    }) =>
                 {
                     let sym = nfc_normalize(self.str_from(start));
                     let span = self.mk_sp(start, self.pos);
-                    self.psess.bad_unicode_identifiers.borrow_mut().entry(sym).or_default()
+                    self.psess
+                        .bad_unicode_identifiers
+                        .borrow_mut()
+                        .entry(sym)
+                        .or_default()
                         .push(span);
                     token::Ident(sym, IdentIsRaw::No)
                 }
@@ -249,9 +257,9 @@ impl<'psess, 'src> StringReader<'psess, 'src> {
                     let suffix = if suffix_start < self.pos {
                         let string = self.str_from(suffix_start);
                         if string == "_" {
-                            self
-                                .dcx()
-                                .emit_err(errors::UnderscoreLiteralSuffix { span: self.mk_sp(suffix_start, self.pos) });
+                            self.dcx().emit_err(errors::UnderscoreLiteralSuffix {
+                                span: self.mk_sp(suffix_start, self.pos),
+                            });
                             None
                         } else {
                             Some(Symbol::intern(string))
@@ -269,12 +277,47 @@ impl<'psess, 'src> StringReader<'psess, 'src> {
                     self.last_lifetime = Some(self.mk_sp(start, start + BytePos(1)));
                     if starts_with_number {
                         let span = self.mk_sp(start, self.pos);
-                        self.dcx().struct_err("lifetimes cannot start with a number")
+                        self.dcx()
+                            .struct_err("lifetimes cannot start with a number")
                             .with_span(span)
                             .stash(span, StashKey::LifetimeIsChar);
                     }
                     let ident = Symbol::intern(lifetime_name);
-                    token::Lifetime(ident)
+                    token::Lifetime(ident, IdentIsRaw::No)
+                }
+                rustc_lexer::TokenKind::RawLifetime => {
+                    self.last_lifetime = Some(self.mk_sp(start, start + BytePos(1)));
+
+                    let ident_start = start + BytePos(3);
+                    let prefix_span = self.mk_sp(start, ident_start);
+
+                    if prefix_span.at_least_rust_2021() {
+                        let lifetime_name_without_tick = self.str_from(ident_start);
+                        // Put the `'` back onto the lifetime name.
+                        let mut lifetime_name = String::with_capacity(lifetime_name_without_tick.len() + 1);
+                        lifetime_name.push('\'');
+                        lifetime_name += lifetime_name_without_tick;
+                        let sym = Symbol::intern(&lifetime_name);
+
+                        token::Lifetime(sym, IdentIsRaw::Yes)
+                    } else {
+                        // Otherwise, this should be parsed like `'r`. Warn about it though.
+                        self.psess.buffer_lint(
+                            RUST_2021_PREFIXES_INCOMPATIBLE_SYNTAX,
+                            prefix_span,
+                            ast::CRATE_NODE_ID,
+                            BuiltinLintDiag::RawPrefix(prefix_span),
+                        );
+
+                        // Reset the state so we just lex the `'r`.
+                        let lt_start = start + BytePos(2);
+                        self.pos = lt_start;
+                        self.cursor = Cursor::new(&str_before[2 as usize..]);
+
+                        let lifetime_name = self.str_from(start);
+                        let ident = Symbol::intern(lifetime_name);
+                        token::Lifetime(ident, IdentIsRaw::No)
+                    }
                 }
                 rustc_lexer::TokenKind::Semi => token::Semi,
                 rustc_lexer::TokenKind::Comma => token::Comma,
@@ -331,16 +374,19 @@ impl<'psess, 'src> StringReader<'psess, 'src> {
                     // first remove compound tokens like `<<` from `rustc_lexer`, and then add
                     // fancier error recovery to it, as there will be less overall work to do this
                     // way.
-                    let (token, sugg) = unicode_chars::check_for_substitution(self, start, c, repeats+1);
+                    let (token, sugg) =
+                        unicode_chars::check_for_substitution(self, start, c, repeats + 1);
                     self.dcx().emit_err(errors::UnknownTokenStart {
                         span: self.mk_sp(start, self.pos + Pos::from_usize(repeats * c.len_utf8())),
                         escaped: escaped_char(c),
                         sugg,
-                        null: if c == '\x00' {Some(errors::UnknownTokenNull)} else {None},
+                        null: if c == '\x00' { Some(errors::UnknownTokenNull) } else { None },
                         repeat: if repeats > 0 {
                             swallow_next_invalid = repeats;
                             Some(errors::UnknownTokenRepeat { repeats })
-                        } else {None}
+                        } else {
+                            None
+                        },
                     });
 
                     if let Some(token) = token {
@@ -699,7 +745,7 @@ impl<'psess, 'src> StringReader<'psess, 'src> {
 
         let expn_data = prefix_span.ctxt().outer_expn_data();
 
-        if expn_data.edition >= Edition::Edition2021 {
+        if expn_data.edition.at_least_rust_2021() {
             // In Rust 2021, this is a hard error.
             let sugg = if prefix == "rb" {
                 Some(errors::UnknownPrefixSugg::UseBr(prefix_span))
diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs
index 49df2811d52..205cca830b2 100644
--- a/compiler/rustc_parse/src/parser/attr_wrapper.rs
+++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs
@@ -1,3 +1,4 @@
+use std::borrow::Cow;
 use std::{iter, mem};
 
 use rustc_ast::token::{Delimiter, Token, TokenKind};
@@ -6,6 +7,7 @@ use rustc_ast::tokenstream::{
     Spacing, ToAttrTokenStream,
 };
 use rustc_ast::{self as ast, AttrVec, Attribute, HasAttrs, HasTokens};
+use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::PResult;
 use rustc_session::parse::ParseSess;
 use rustc_span::{sym, Span, DUMMY_SP};
@@ -134,9 +136,8 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl {
                 node_replacements.array_windows()
             {
                 assert!(
-                    node_range.0.end <= next_node_range.0.start
-                        || node_range.0.end >= next_node_range.0.end,
-                    "Node ranges should be disjoint or nested: ({:?}, {:?}) ({:?}, {:?})",
+                    node_range.0.end <= next_node_range.0.start,
+                    "Node ranges should be disjoint: ({:?}, {:?}) ({:?}, {:?})",
                     node_range,
                     tokens,
                     next_node_range,
@@ -144,20 +145,8 @@ impl ToAttrTokenStream for LazyAttrTokenStreamImpl {
                 );
             }
 
-            // Process the replace ranges, starting from the highest start
-            // position and working our way back. If have tokens like:
-            //
-            // `#[cfg(FALSE)] struct Foo { #[cfg(FALSE)] field: bool }`
-            //
-            // Then we will generate replace ranges for both
-            // the `#[cfg(FALSE)] field: bool` and the entire
-            // `#[cfg(FALSE)] struct Foo { #[cfg(FALSE)] field: bool }`
-            //
-            // By starting processing from the replace range with the greatest
-            // start position, we ensure that any (outer) replace range which
-            // encloses another (inner) replace range will fully overwrite the
-            // inner range's replacement.
-            for (node_range, target) in node_replacements.into_iter().rev() {
+            // Process the replace ranges.
+            for (node_range, target) in node_replacements.into_iter() {
                 assert!(
                     !node_range.0.is_empty(),
                     "Cannot replace an empty node range: {:?}",
@@ -234,6 +223,8 @@ impl<'a> Parser<'a> {
         force_collect: ForceCollect,
         f: impl FnOnce(&mut Self, AttrVec) -> PResult<'a, (R, Trailing, UsePreAttrPos)>,
     ) -> PResult<'a, R> {
+        let possible_capture_mode = self.capture_cfg;
+
         // We must collect if anything could observe the collected tokens, i.e.
         // if any of the following conditions hold.
         // - We are force collecting tokens (because force collection requires
@@ -244,9 +235,9 @@ impl<'a> Parser<'a> {
             // - Our target supports custom inner attributes (custom
             //   inner attribute invocation might require token capturing).
             || R::SUPPORTS_CUSTOM_INNER_ATTRS
-            // - We are in `capture_cfg` mode (which requires tokens if
+            // - We are in "possible capture mode" (which requires tokens if
             //   the parsed node has `#[cfg]` or `#[cfg_attr]` attributes).
-            || self.capture_cfg;
+            || possible_capture_mode;
         if !needs_collection {
             return Ok(f(self, attrs.attrs)?.0);
         }
@@ -267,18 +258,48 @@ impl<'a> Parser<'a> {
             res?
         };
 
-        // When we're not in `capture_cfg` mode, then skip collecting and
-        // return early if either of the following conditions hold.
         // - `None`: Our target doesn't support tokens at all (e.g. `NtIdent`).
+        // - `Some(None)`: Our target supports tokens and has none.
         // - `Some(Some(_))`: Our target already has tokens set (e.g. we've
-        //   parsed something like `#[my_attr] $item`). The actual parsing code
-        //   takes care of prepending any attributes to the nonterminal, so we
-        //   don't need to modify the already captured tokens.
+        //   parsed something like `#[my_attr] $item`).
+        let ret_can_hold_tokens = matches!(ret.tokens_mut(), Some(None));
+
+        // Ignore any attributes we've previously processed. This happens when
+        // an inner call to `collect_tokens` returns an AST node and then an
+        // outer call ends up with the same AST node without any additional
+        // wrapping layer.
+        let mut seen_indices = FxHashSet::default();
+        for (i, attr) in ret.attrs().iter().enumerate() {
+            let is_unseen = self.capture_state.seen_attrs.insert(attr.id);
+            if !is_unseen {
+                seen_indices.insert(i);
+            }
+        }
+        let ret_attrs: Cow<'_, [Attribute]> =
+            if seen_indices.is_empty() {
+                Cow::Borrowed(ret.attrs())
+            } else {
+                let ret_attrs =
+                    ret.attrs()
+                        .iter()
+                        .enumerate()
+                        .filter_map(|(i, attr)| {
+                            if seen_indices.contains(&i) { None } else { Some(attr.clone()) }
+                        })
+                        .collect();
+                Cow::Owned(ret_attrs)
+            };
+
+        // When we're not in "definite capture mode", then skip collecting and
+        // return early if `ret` doesn't support tokens or already has some.
         //
         // Note that this check is independent of `force_collect`. There's no
         // need to collect tokens when we don't support tokens or already have
         // tokens.
-        if !self.capture_cfg && matches!(ret.tokens_mut(), None | Some(Some(_))) {
+        let definite_capture_mode = self.capture_cfg
+            && matches!(self.capture_state.capturing, Capturing::Yes)
+            && has_cfg_or_cfg_attr(&ret_attrs);
+        if !definite_capture_mode && !ret_can_hold_tokens {
             return Ok(ret);
         }
 
@@ -297,12 +318,12 @@ impl<'a> Parser<'a> {
             //   outer and inner attributes. So this check is more precise than
             //   the earlier `needs_tokens` check, and we don't need to
             //   check `R::SUPPORTS_CUSTOM_INNER_ATTRS`.)
-            || needs_tokens(ret.attrs())
-            // - We are in `capture_cfg` mode and there are `#[cfg]` or
-            //   `#[cfg_attr]` attributes. (During normal non-`capture_cfg`
-            //   parsing, we don't need any special capturing for those
-            //   attributes, because they're builtin.)
-            || (self.capture_cfg && has_cfg_or_cfg_attr(ret.attrs()));
+            || needs_tokens(&ret_attrs)
+            // - We are in "definite capture mode", which requires that there
+            //   are `#[cfg]` or `#[cfg_attr]` attributes. (During normal
+            //   non-`capture_cfg` parsing, we don't need any special capturing
+            //   for those attributes, because they're builtin.)
+            || definite_capture_mode;
         if !needs_collection {
             return Ok(ret);
         }
@@ -336,7 +357,7 @@ impl<'a> Parser<'a> {
         // `Parser::parse_inner_attributes`, and pair them in a `ParserReplacement` with `None`,
         // which means the relevant tokens will be removed. (More details below.)
         let mut inner_attr_parser_replacements = Vec::new();
-        for attr in ret.attrs() {
+        for attr in ret_attrs.iter() {
             if attr.style == ast::AttrStyle::Inner {
                 if let Some(inner_attr_parser_range) =
                     self.capture_state.inner_attr_parser_ranges.remove(&attr.id)
@@ -359,11 +380,10 @@ impl<'a> Parser<'a> {
             // from `ParserRange` form to `NodeRange` form. We will perform the actual
             // replacement only when we convert the `LazyAttrTokenStream` to an
             // `AttrTokenStream`.
-            self.capture_state.parser_replacements
-                [parser_replacements_start..parser_replacements_end]
-                .iter()
-                .cloned()
-                .chain(inner_attr_parser_replacements.iter().cloned())
+            self.capture_state
+                .parser_replacements
+                .drain(parser_replacements_start..parser_replacements_end)
+                .chain(inner_attr_parser_replacements.into_iter())
                 .map(|(parser_range, data)| {
                     (NodeRange::new(parser_range, collect_pos.start_pos), data)
                 })
@@ -399,20 +419,12 @@ impl<'a> Parser<'a> {
             break_last_token: self.break_last_token,
             node_replacements,
         });
+        let mut tokens_used = false;
 
-        // If we support tokens and don't already have them, store the newly captured tokens.
-        if let Some(target_tokens @ None) = ret.tokens_mut() {
-            *target_tokens = Some(tokens.clone());
-        }
-
-        // If `capture_cfg` is set and we're inside a recursive call to
-        // `collect_tokens`, then we need to register a replace range if we
-        // have `#[cfg]` or `#[cfg_attr]`. This allows us to run eager
-        // cfg-expansion on the captured token stream.
-        if self.capture_cfg
-            && matches!(self.capture_state.capturing, Capturing::Yes)
-            && has_cfg_or_cfg_attr(ret.attrs())
-        {
+        // If in "definite capture mode" we need to register a replace range
+        // for the `#[cfg]` and/or `#[cfg_attr]` attrs. This allows us to run
+        // eager cfg-expansion on the captured token stream.
+        if definite_capture_mode {
             assert!(!self.break_last_token, "Should not have unglued last token with cfg attr");
 
             // What is the status here when parsing the example code at the top of this method?
@@ -429,7 +441,9 @@ impl<'a> Parser<'a> {
             // cfg-expand this AST node.
             let start_pos =
                 if has_outer_attrs { attrs.start_pos.unwrap() } else { collect_pos.start_pos };
-            let target = AttrsTarget { attrs: ret.attrs().iter().cloned().collect(), tokens };
+            let target =
+                AttrsTarget { attrs: ret_attrs.iter().cloned().collect(), tokens: tokens.clone() };
+            tokens_used = true;
             self.capture_state
                 .parser_replacements
                 .push((ParserRange(start_pos..end_pos), Some(target)));
@@ -438,7 +452,16 @@ impl<'a> Parser<'a> {
             // the outermost call to this method.
             self.capture_state.parser_replacements.clear();
             self.capture_state.inner_attr_parser_ranges.clear();
+            self.capture_state.seen_attrs.clear();
         }
+
+        // If we support tokens and don't already have them, store the newly captured tokens.
+        if let Some(target_tokens @ None) = ret.tokens_mut() {
+            tokens_used = true;
+            *target_tokens = Some(tokens);
+        }
+
+        assert!(tokens_used); // check we didn't create `tokens` unnecessarily
         Ok(ret)
     }
 }
@@ -510,9 +533,11 @@ fn make_attr_token_stream(
 }
 
 /// Tokens are needed if:
-/// - any non-single-segment attributes (other than doc comments) are present; or
-/// - any `cfg_attr` attributes are present;
-/// - any single-segment, non-builtin attributes are present.
+/// - any non-single-segment attributes (other than doc comments) are present,
+///   e.g. `rustfmt::skip`; or
+/// - any `cfg_attr` attributes are present; or
+/// - any single-segment, non-builtin attributes are present, e.g. `derive`,
+///   `test`, `global_allocator`.
 fn needs_tokens(attrs: &[ast::Attribute]) -> bool {
     attrs.iter().any(|attr| match attr.ident() {
         None => !attr.is_doc_comment(),
diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs
index fcdc10c0837..f6f66821df7 100644
--- a/compiler/rustc_parse/src/parser/diagnostics.rs
+++ b/compiler/rustc_parse/src/parser/diagnostics.rs
@@ -19,8 +19,9 @@ use rustc_errors::{
     Subdiagnostic,
 };
 use rustc_session::errors::ExprParenthesesNeeded;
+use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::source_map::Spanned;
-use rustc_span::symbol::{kw, sym, Ident};
+use rustc_span::symbol::{kw, sym, AllKeywords, Ident};
 use rustc_span::{BytePos, Span, SpanSnippetError, Symbol, DUMMY_SP};
 use thin_vec::{thin_vec, ThinVec};
 use tracing::{debug, trace};
@@ -203,6 +204,37 @@ impl std::fmt::Display for UnaryFixity {
     }
 }
 
+#[derive(Debug, rustc_macros::Subdiagnostic)]
+#[suggestion(
+    parse_misspelled_kw,
+    applicability = "machine-applicable",
+    code = "{similar_kw}",
+    style = "verbose"
+)]
+struct MisspelledKw {
+    similar_kw: String,
+    #[primary_span]
+    span: Span,
+    is_incorrect_case: bool,
+}
+
+/// Checks if the given `lookup` identifier is similar to any keyword symbol in `candidates`.
+fn find_similar_kw(lookup: Ident, candidates: &[Symbol]) -> Option<MisspelledKw> {
+    let lowercase = lookup.name.as_str().to_lowercase();
+    let lowercase_sym = Symbol::intern(&lowercase);
+    if candidates.contains(&lowercase_sym) {
+        Some(MisspelledKw { similar_kw: lowercase, span: lookup.span, is_incorrect_case: true })
+    } else if let Some(similar_sym) = find_best_match_for_name(candidates, lookup.name, None) {
+        Some(MisspelledKw {
+            similar_kw: similar_sym.to_string(),
+            span: lookup.span,
+            is_incorrect_case: false,
+        })
+    } else {
+        None
+    }
+}
+
 struct MultiSugg {
     msg: String,
     patches: Vec<(Span, String)>,
@@ -638,9 +670,9 @@ impl<'a> Parser<'a> {
             let concat = Symbol::intern(&format!("{prev}{cur}"));
             let ident = Ident::new(concat, DUMMY_SP);
             if ident.is_used_keyword() || ident.is_reserved() || ident.is_raw_guess() {
-                let span = self.prev_token.span.to(self.token.span);
+                let concat_span = self.prev_token.span.to(self.token.span);
                 err.span_suggestion_verbose(
-                    span,
+                    concat_span,
                     format!("consider removing the space to spell keyword `{concat}`"),
                     concat,
                     Applicability::MachineApplicable,
@@ -741,9 +773,55 @@ impl<'a> Parser<'a> {
             err.span_label(sp, label_exp);
             err.span_label(self.token.span, "unexpected token");
         }
+
+        // Check for misspelled keywords if there are no suggestions added to the diagnostic.
+        if err.suggestions.as_ref().is_ok_and(|code_suggestions| code_suggestions.is_empty()) {
+            self.check_for_misspelled_kw(&mut err, &expected);
+        }
         Err(err)
     }
 
+    /// Checks if the current token or the previous token are misspelled keywords
+    /// and adds a helpful suggestion.
+    fn check_for_misspelled_kw(&self, err: &mut Diag<'_>, expected: &[TokenType]) {
+        let Some((curr_ident, _)) = self.token.ident() else {
+            return;
+        };
+        let expected_tokens: &[TokenType] =
+            expected.len().checked_sub(10).map_or(&expected, |index| &expected[index..]);
+        let expected_keywords: Vec<Symbol> = expected_tokens
+            .iter()
+            .filter_map(|token| if let TokenType::Keyword(kw) = token { Some(*kw) } else { None })
+            .collect();
+
+        // When there are a few keywords in the last ten elements of `self.expected_tokens` and the current
+        // token is an identifier, it's probably a misspelled keyword.
+        // This handles code like `async Move {}`, misspelled `if` in match guard, misspelled `else` in `if`-`else`
+        // and mispelled `where` in a where clause.
+        if !expected_keywords.is_empty()
+            && !curr_ident.is_used_keyword()
+            && let Some(misspelled_kw) = find_similar_kw(curr_ident, &expected_keywords)
+        {
+            err.subdiagnostic(misspelled_kw);
+        } else if let Some((prev_ident, _)) = self.prev_token.ident()
+            && !prev_ident.is_used_keyword()
+        {
+            // We generate a list of all keywords at runtime rather than at compile time
+            // so that it gets generated only when the diagnostic needs it.
+            // Also, it is unlikely that this list is generated multiple times because the
+            // parser halts after execution hits this path.
+            let all_keywords = AllKeywords::new().collect_used(|| prev_ident.span.edition());
+
+            // Otherwise, check the previous token with all the keywords as possible candidates.
+            // This handles code like `Struct Human;` and `While a < b {}`.
+            // We check the previous token only when the current token is an identifier to avoid false
+            // positives like suggesting keyword `for` for `extern crate foo {}`.
+            if let Some(misspelled_kw) = find_similar_kw(prev_ident, &all_keywords) {
+                err.subdiagnostic(misspelled_kw);
+            }
+        }
+    }
+
     /// The user has written `#[attr] expr` which is unsupported. (#106020)
     pub(super) fn attr_on_non_tail_expr(&self, expr: &Expr) -> ErrorGuaranteed {
         // Missing semicolon typo error.
@@ -846,6 +924,7 @@ impl<'a> Parser<'a> {
                 );
             }
         }
+
         err.emit()
     }
 
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index 84684e808d9..ecc4cd96faf 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -2050,7 +2050,7 @@ impl<'a> Parser<'a> {
         };
         // On an error path, eagerly consider a lifetime to be an unclosed character lit, if that
         // makes sense.
-        if let Some(ident) = self.token.lifetime()
+        if let Some((ident, IdentIsRaw::No)) = self.token.lifetime()
             && could_be_unclosed_char_literal(ident)
         {
             let lt = self.expect_lifetime();
@@ -2925,9 +2925,9 @@ impl<'a> Parser<'a> {
     }
 
     pub(crate) fn eat_label(&mut self) -> Option<Label> {
-        if let Some(ident) = self.token.lifetime() {
+        if let Some((ident, is_raw)) = self.token.lifetime() {
             // Disallow `'fn`, but with a better error message than `expect_lifetime`.
-            if ident.without_first_quote().is_reserved() {
+            if matches!(is_raw, IdentIsRaw::No) && ident.without_first_quote().is_reserved() {
                 self.dcx().emit_err(errors::InvalidLabel { span: ident.span, name: ident.name });
             }
 
diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs
index 61e3fa2e6c5..3ee6e742d1b 100644
--- a/compiler/rustc_parse/src/parser/mod.rs
+++ b/compiler/rustc_parse/src/parser/mod.rs
@@ -35,6 +35,7 @@ use rustc_ast_pretty::pprust;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::{Applicability, Diag, FatalError, MultiSpan, PResult};
+use rustc_index::interval::IntervalSet;
 use rustc_session::parse::ParseSess;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{Span, DUMMY_SP};
@@ -183,7 +184,7 @@ pub struct Parser<'a> {
 // This type is used a lot, e.g. it's cloned when matching many declarative macro rules with nonterminals. Make sure
 // it doesn't unintentionally get bigger.
 #[cfg(target_pointer_width = "64")]
-rustc_data_structures::static_assert_size!(Parser<'_>, 256);
+rustc_data_structures::static_assert_size!(Parser<'_>, 288);
 
 /// Stores span information about a closure.
 #[derive(Clone, Debug)]
@@ -238,7 +239,8 @@ impl NodeRange {
     // is the position of the function's start token. This gives
     // `NodeRange(10..15)`.
     fn new(ParserRange(parser_range): ParserRange, start_pos: u32) -> NodeRange {
-        assert!(parser_range.start >= start_pos && parser_range.end >= start_pos);
+        assert!(!parser_range.is_empty());
+        assert!(parser_range.start >= start_pos);
         NodeRange((parser_range.start - start_pos)..(parser_range.end - start_pos))
     }
 }
@@ -260,6 +262,9 @@ struct CaptureState {
     capturing: Capturing,
     parser_replacements: Vec<ParserReplacement>,
     inner_attr_parser_ranges: FxHashMap<AttrId, ParserRange>,
+    // `IntervalSet` is good for perf because attrs are mostly added to this
+    // set in contiguous ranges.
+    seen_attrs: IntervalSet<AttrId>,
 }
 
 /// Iterator over a `TokenStream` that produces `Token`s. It's a bit odd that
@@ -457,6 +462,7 @@ impl<'a> Parser<'a> {
                 capturing: Capturing::No,
                 parser_replacements: Vec::new(),
                 inner_attr_parser_ranges: Default::default(),
+                seen_attrs: IntervalSet::new(u32::MAX as usize),
             },
             current_closure: None,
             recovery: Recovery::Allowed,
@@ -1666,7 +1672,7 @@ enum FlatToken {
 pub enum ParseNtResult {
     Tt(TokenTree),
     Ident(Ident, IdentIsRaw),
-    Lifetime(Ident),
+    Lifetime(Ident, IdentIsRaw),
 
     /// This case will eventually be removed, along with `Token::Interpolate`.
     Nt(Lrc<Nonterminal>),
diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs
index e66d0df012b..44b169c881e 100644
--- a/compiler/rustc_parse/src/parser/nonterminal.rs
+++ b/compiler/rustc_parse/src/parser/nonterminal.rs
@@ -88,7 +88,7 @@ impl<'a> Parser<'a> {
             },
             NonterminalKind::Pat(pat_kind) => token.can_begin_pattern(pat_kind),
             NonterminalKind::Lifetime => match &token.kind {
-                token::Lifetime(_) | token::NtLifetime(..) => true,
+                token::Lifetime(..) | token::NtLifetime(..) => true,
                 _ => false,
             },
             NonterminalKind::TT | NonterminalKind::Item | NonterminalKind::Stmt => {
@@ -171,9 +171,9 @@ impl<'a> Parser<'a> {
             NonterminalKind::Lifetime => {
                 // We want to keep `'keyword` parsing, just like `keyword` is still
                 // an ident for nonterminal purposes.
-                return if let Some(ident) = self.token.lifetime() {
+                return if let Some((ident, is_raw)) = self.token.lifetime() {
                     self.bump();
-                    Ok(ParseNtResult::Lifetime(ident))
+                    Ok(ParseNtResult::Lifetime(ident, is_raw))
                 } else {
                     Err(self.dcx().create_err(UnexpectedNonterminal::Lifetime {
                         span: self.token.span,
diff --git a/compiler/rustc_parse/src/parser/pat.rs b/compiler/rustc_parse/src/parser/pat.rs
index f87b5649654..cbd35ffdfa9 100644
--- a/compiler/rustc_parse/src/parser/pat.rs
+++ b/compiler/rustc_parse/src/parser/pat.rs
@@ -1,6 +1,6 @@
 use rustc_ast::mut_visit::{walk_pat, MutVisitor};
 use rustc_ast::ptr::P;
-use rustc_ast::token::{self, BinOpToken, Delimiter, Token};
+use rustc_ast::token::{self, BinOpToken, Delimiter, IdentIsRaw, Token};
 use rustc_ast::{
     self as ast, AttrVec, BindingMode, ByRef, Expr, ExprKind, MacCall, Mutability, Pat, PatField,
     PatFieldsRest, PatKind, Path, QSelf, RangeEnd, RangeSyntax,
@@ -548,7 +548,7 @@ impl<'a> Parser<'a> {
                     None => PatKind::Path(qself, path),
                 }
             }
-        } else if let Some(lt) = self.token.lifetime()
+        } else if let Some((lt, IdentIsRaw::No)) = self.token.lifetime()
             // In pattern position, we're totally fine with using "next token isn't colon"
             // as a heuristic. We could probably just always try to recover if it's a lifetime,
             // because we never have `'a: label {}` in a pattern position anyways, but it does
@@ -689,7 +689,7 @@ impl<'a> Parser<'a> {
     /// Parse `&pat` / `&mut pat`.
     fn parse_pat_deref(&mut self, expected: Option<Expected>) -> PResult<'a, PatKind> {
         self.expect_and()?;
-        if let Some(lifetime) = self.token.lifetime() {
+        if let Some((lifetime, _)) = self.token.lifetime() {
             self.bump(); // `'a`
 
             self.dcx().emit_err(UnexpectedLifetimeInPattern {
diff --git a/compiler/rustc_parse/src/parser/ty.rs b/compiler/rustc_parse/src/parser/ty.rs
index 52f0b1c1b04..dd1cc75c7ff 100644
--- a/compiler/rustc_parse/src/parser/ty.rs
+++ b/compiler/rustc_parse/src/parser/ty.rs
@@ -1,5 +1,5 @@
 use rustc_ast::ptr::P;
-use rustc_ast::token::{self, BinOpToken, Delimiter, Token, TokenKind};
+use rustc_ast::token::{self, BinOpToken, Delimiter, IdentIsRaw, Token, TokenKind};
 use rustc_ast::util::case::Case;
 use rustc_ast::{
     self as ast, BareFnTy, BoundAsyncness, BoundConstness, BoundPolarity, FnRetTy, GenericBound,
@@ -1285,8 +1285,9 @@ impl<'a> Parser<'a> {
 
     /// Parses a single lifetime `'a` or panics.
     pub(super) fn expect_lifetime(&mut self) -> Lifetime {
-        if let Some(ident) = self.token.lifetime() {
-            if ident.without_first_quote().is_reserved()
+        if let Some((ident, is_raw)) = self.token.lifetime() {
+            if matches!(is_raw, IdentIsRaw::No)
+                && ident.without_first_quote().is_reserved()
                 && ![kw::UnderscoreLifetime, kw::StaticLifetime].contains(&ident.name)
             {
                 self.dcx().emit_err(errors::KeywordLifetime { span: ident.span });
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index 21478a44b0e..e41f89a3c9d 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -125,7 +125,9 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                 [sym::inline, ..] => self.check_inline(hir_id, attr, span, target),
                 [sym::coverage, ..] => self.check_coverage(attr, span, target),
                 [sym::optimize, ..] => self.check_optimize(hir_id, attr, target),
-                [sym::no_sanitize, ..] => self.check_no_sanitize(hir_id, attr, span, target),
+                [sym::no_sanitize, ..] => {
+                    self.check_applied_to_fn_or_method(hir_id, attr, span, target)
+                }
                 [sym::non_exhaustive, ..] => self.check_non_exhaustive(hir_id, attr, span, target),
                 [sym::marker, ..] => self.check_marker(hir_id, attr, span, target),
                 [sym::target_feature, ..] => {
@@ -166,10 +168,13 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                     self.check_rustc_legacy_const_generics(hir_id, attr, span, target, item)
                 }
                 [sym::rustc_lint_query_instability, ..] => {
-                    self.check_rustc_lint_query_instability(hir_id, attr, span, target)
+                    self.check_applied_to_fn_or_method(hir_id, attr, span, target)
+                }
+                [sym::rustc_lint_untracked_query_information, ..] => {
+                    self.check_applied_to_fn_or_method(hir_id, attr, span, target)
                 }
                 [sym::rustc_lint_diagnostics, ..] => {
-                    self.check_rustc_lint_diagnostics(hir_id, attr, span, target)
+                    self.check_applied_to_fn_or_method(hir_id, attr, span, target)
                 }
                 [sym::rustc_lint_opt_ty, ..] => self.check_rustc_lint_opt_ty(attr, span, target),
                 [sym::rustc_lint_opt_deny_field_access, ..] => {
@@ -452,11 +457,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
         }
     }
 
-    /// Checks that `#[no_sanitize(..)]` is applied to a function or method.
-    fn check_no_sanitize(&self, hir_id: HirId, attr: &Attribute, span: Span, target: Target) {
-        self.check_applied_to_fn_or_method(hir_id, attr, span, target)
-    }
-
     fn check_generic_attr(
         &self,
         hir_id: HirId,
@@ -1635,30 +1635,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
         }
     }
 
-    /// Checks that the `#[rustc_lint_query_instability]` attribute is only applied to a function
-    /// or method.
-    fn check_rustc_lint_query_instability(
-        &self,
-        hir_id: HirId,
-        attr: &Attribute,
-        span: Span,
-        target: Target,
-    ) {
-        self.check_applied_to_fn_or_method(hir_id, attr, span, target)
-    }
-
-    /// Checks that the `#[rustc_lint_diagnostics]` attribute is only applied to a function or
-    /// method.
-    fn check_rustc_lint_diagnostics(
-        &self,
-        hir_id: HirId,
-        attr: &Attribute,
-        span: Span,
-        target: Target,
-    ) {
-        self.check_applied_to_fn_or_method(hir_id, attr, span, target)
-    }
-
     /// Checks that the `#[rustc_lint_opt_ty]` attribute is only applied to a struct.
     fn check_rustc_lint_opt_ty(&self, attr: &Attribute, span: Span, target: Target) {
         match target {
diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs
index 79c42456cf8..917cb81aa51 100644
--- a/compiler/rustc_resolve/src/late.rs
+++ b/compiler/rustc_resolve/src/late.rs
@@ -2062,7 +2062,10 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
                                 lint::builtin::ELIDED_NAMED_LIFETIMES,
                                 missing.id_for_lint,
                                 missing.span,
-                                BuiltinLintDiag::ElidedIsStatic { elided: missing.span },
+                                BuiltinLintDiag::ElidedNamedLifetimes {
+                                    elided: (missing.span, missing.kind),
+                                    resolution: lint::ElidedLifetimeResolution::Static,
+                                },
                             );
                         }
                     }
@@ -2072,9 +2075,12 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
                             lint::builtin::ELIDED_NAMED_LIFETIMES,
                             missing.id_for_lint,
                             missing.span,
-                            BuiltinLintDiag::ElidedIsParam {
-                                elided: missing.span,
-                                param: (tcx.item_name(param.into()), tcx.source_span(param)),
+                            BuiltinLintDiag::ElidedNamedLifetimes {
+                                elided: (missing.span, missing.kind),
+                                resolution: lint::ElidedLifetimeResolution::Param(
+                                    tcx.item_name(param.into()),
+                                    tcx.source_span(param),
+                                ),
                             },
                         );
                     }
diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs
index 908d50a041e..f3e3b36111c 100644
--- a/compiler/rustc_session/src/config.rs
+++ b/compiler/rustc_session/src/config.rs
@@ -842,6 +842,11 @@ pub struct NextSolverConfig {
     /// This is only `true` if `coherence` is also enabled.
     pub globally: bool,
 }
+impl Default for NextSolverConfig {
+    fn default() -> Self {
+        NextSolverConfig { coherence: true, globally: false }
+    }
+}
 
 #[derive(Clone)]
 pub enum Input {
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index 45339c23ea3..a57dc80b316 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -403,7 +403,7 @@ mod desc {
     pub(crate) const parse_unpretty: &str = "`string` or `string=string`";
     pub(crate) const parse_treat_err_as_bug: &str = "either no value or a non-negative number";
     pub(crate) const parse_next_solver_config: &str =
-        "a comma separated list of solver configurations: `globally` (default), and `coherence`";
+        "either `globally` (when used without an argument), `coherence` (default) or `no`";
     pub(crate) const parse_lto: &str =
         "either a boolean (`yes`, `no`, `on`, `off`, etc), `thin`, `fat`, or omitted";
     pub(crate) const parse_linker_plugin_lto: &str =
@@ -1105,27 +1105,16 @@ mod parse {
         }
     }
 
-    pub(crate) fn parse_next_solver_config(
-        slot: &mut Option<NextSolverConfig>,
-        v: Option<&str>,
-    ) -> bool {
+    pub(crate) fn parse_next_solver_config(slot: &mut NextSolverConfig, v: Option<&str>) -> bool {
         if let Some(config) = v {
-            let mut coherence = false;
-            let mut globally = true;
-            for c in config.split(',') {
-                match c {
-                    "globally" => globally = true,
-                    "coherence" => {
-                        globally = false;
-                        coherence = true;
-                    }
-                    _ => return false,
-                }
-            }
-
-            *slot = Some(NextSolverConfig { coherence: coherence || globally, globally });
+            *slot = match config {
+                "no" => NextSolverConfig { coherence: false, globally: false },
+                "coherence" => NextSolverConfig { coherence: true, globally: false },
+                "globally" => NextSolverConfig { coherence: true, globally: true },
+                _ => return false,
+            };
         } else {
-            *slot = Some(NextSolverConfig { coherence: true, globally: true });
+            *slot = NextSolverConfig { coherence: true, globally: true };
         }
 
         true
@@ -1878,7 +1867,7 @@ options! {
         "the size at which the `large_assignments` lint starts to be emitted"),
     mutable_noalias: bool = (true, parse_bool, [TRACKED],
         "emit noalias metadata for mutable references (default: yes)"),
-    next_solver: Option<NextSolverConfig> = (None, parse_next_solver_config, [TRACKED],
+    next_solver: NextSolverConfig = (NextSolverConfig::default(), parse_next_solver_config, [TRACKED],
         "enable and configure the next generation trait solver used by rustc"),
     nll_facts: bool = (false, parse_bool, [UNTRACKED],
         "dump facts from NLL analysis into side files (default: no)"),
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 36b31d10c4d..7572d57309c 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -20,7 +20,8 @@ mod tests;
 
 // The proc macro code for this is in `compiler/rustc_macros/src/symbols.rs`.
 symbols! {
-    // If you modify this list, adjust `is_special` and `is_used_keyword`/`is_unused_keyword`.
+    // If you modify this list, adjust `is_special`, `is_used_keyword`/`is_unused_keyword`
+    // and `AllKeywords`.
     // But this should rarely be necessary if the keywords are kept in alphabetic order.
     Keywords {
         // Special reserved identifiers used internally for elided lifetimes,
@@ -1654,6 +1655,7 @@ symbols! {
         rustc_lint_opt_deny_field_access,
         rustc_lint_opt_ty,
         rustc_lint_query_instability,
+        rustc_lint_untracked_query_information,
         rustc_macro_transparency,
         rustc_main,
         rustc_mir,
@@ -2578,3 +2580,42 @@ impl Ident {
         self.name.can_be_raw() && self.is_reserved()
     }
 }
+
+/// An iterator over all the keywords in Rust.
+#[derive(Copy, Clone)]
+pub struct AllKeywords {
+    curr_idx: u32,
+    end_idx: u32,
+}
+
+impl AllKeywords {
+    /// Initialize a new iterator over all the keywords.
+    ///
+    /// *Note:* Please update this if a new keyword is added beyond the current
+    /// range.
+    pub fn new() -> Self {
+        AllKeywords { curr_idx: kw::Empty.as_u32(), end_idx: kw::Yeet.as_u32() }
+    }
+
+    /// Collect all the keywords in a given edition into a vector.
+    pub fn collect_used(&self, edition: impl Copy + FnOnce() -> Edition) -> Vec<Symbol> {
+        self.filter(|&keyword| {
+            keyword.is_used_keyword_always() || keyword.is_used_keyword_conditional(edition)
+        })
+        .collect()
+    }
+}
+
+impl Iterator for AllKeywords {
+    type Item = Symbol;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.curr_idx <= self.end_idx {
+            let keyword = Symbol::new(self.curr_idx);
+            self.curr_idx += 1;
+            Some(keyword)
+        } else {
+            None
+        }
+    }
+}
diff --git a/compiler/rustc_target/src/lib.rs b/compiler/rustc_target/src/lib.rs
index d7c66d6f5e3..2121c4110dd 100644
--- a/compiler/rustc_target/src/lib.rs
+++ b/compiler/rustc_target/src/lib.rs
@@ -9,7 +9,6 @@
 
 // tidy-alphabetical-start
 #![allow(internal_features)]
-#![cfg_attr(bootstrap, feature(min_exhaustive_patterns))]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
 #![feature(assert_matches)]
diff --git a/compiler/rustc_target/src/spec/base/apple/mod.rs b/compiler/rustc_target/src/spec/base/apple/mod.rs
index b752a344395..b1fe49f76ca 100644
--- a/compiler/rustc_target/src/spec/base/apple/mod.rs
+++ b/compiler/rustc_target/src/spec/base/apple/mod.rs
@@ -1,5 +1,6 @@
 use std::borrow::Cow;
 use std::env;
+use std::num::ParseIntError;
 
 use crate::spec::{
     add_link_args, add_link_args_iter, cvs, Cc, DebuginfoKind, FramePointer, LinkArgs,
@@ -39,6 +40,25 @@ impl Arch {
         }
     }
 
+    /// The architecture name to forward to the linker.
+    fn ld_arch(self) -> &'static str {
+        // Supported architecture names can be found in the source:
+        // https://github.com/apple-oss-distributions/ld64/blob/ld64-951.9/src/abstraction/MachOFileAbstraction.hpp#L578-L648
+        match self {
+            Armv7k => "armv7k",
+            Armv7s => "armv7s",
+            Arm64 => "arm64",
+            Arm64e => "arm64e",
+            Arm64_32 => "arm64_32",
+            // ld64 doesn't understand i686, so fall back to i386 instead
+            //
+            // Same story when linking with cc, since that ends up invoking ld64.
+            I386 | I686 => "i386",
+            X86_64 => "x86_64",
+            X86_64h => "x86_64h",
+        }
+    }
+
     pub(crate) fn target_arch(self) -> Cow<'static, str> {
         Cow::Borrowed(match self {
             Armv7k | Armv7s => "arm",
@@ -97,53 +117,85 @@ impl TargetAbi {
 }
 
 fn pre_link_args(os: &'static str, arch: Arch, abi: TargetAbi) -> LinkArgs {
+    // From the man page for ld64 (`man ld`):
+    // > The linker accepts universal (multiple-architecture) input files,
+    // > but always creates a "thin" (single-architecture), standard Mach-O
+    // > output file. The architecture for the output file is specified using
+    // > the -arch option.
+    //
+    // The linker has heuristics to determine the desired architecture, but to
+    // be safe, and to avoid a warning, we set the architecture explicitly.
+    let mut args =
+        TargetOptions::link_args(LinkerFlavor::Darwin(Cc::No, Lld::No), &["-arch", arch.ld_arch()]);
+
+    // From the man page for ld64 (`man ld`):
+    // > This is set to indicate the platform, oldest supported version of
+    // > that platform that output is to be used on, and the SDK that the
+    // > output was built against. platform [...] may be one of the following
+    // > strings:
+    // > - macos
+    // > - ios
+    // > - tvos
+    // > - watchos
+    // > - bridgeos
+    // > - visionos
+    // > - xros
+    // > - mac-catalyst
+    // > - ios-simulator
+    // > - tvos-simulator
+    // > - watchos-simulator
+    // > - visionos-simulator
+    // > - xros-simulator
+    // > - driverkit
+    //
+    // Like with `-arch`, the linker can figure out the platform versions
+    // itself from the binaries being linked, but to be safe, we specify the
+    // desired versions here explicitly.
     let platform_name: StaticCow<str> = match abi {
         TargetAbi::Normal => os.into(),
         TargetAbi::Simulator => format!("{os}-simulator").into(),
         TargetAbi::MacCatalyst => "mac-catalyst".into(),
     };
-
     let min_version: StaticCow<str> = {
-        let (major, minor) = match os {
-            "ios" => ios_deployment_target(arch, abi.target_abi()),
-            "tvos" => tvos_deployment_target(),
-            "watchos" => watchos_deployment_target(),
-            "visionos" => visionos_deployment_target(),
-            "macos" => macos_deployment_target(arch),
-            _ => unreachable!(),
-        };
-        format!("{major}.{minor}").into()
+        let (major, minor, patch) = deployment_target(os, arch, abi);
+        format!("{major}.{minor}.{patch}").into()
     };
+    // Lie about the SDK version, we don't know it here
     let sdk_version = min_version.clone();
-
-    let mut args = TargetOptions::link_args(
-        LinkerFlavor::Darwin(Cc::No, Lld::No),
-        &["-arch", arch.target_name(), "-platform_version"],
-    );
     add_link_args_iter(
         &mut args,
         LinkerFlavor::Darwin(Cc::No, Lld::No),
-        [platform_name, min_version, sdk_version].into_iter(),
+        ["-platform_version".into(), platform_name, min_version, sdk_version].into_iter(),
     );
+
     if abi != TargetAbi::MacCatalyst {
+        // CC forwards the `-arch` to the linker, so we use the same value
+        // here intentionally.
         add_link_args(
             &mut args,
             LinkerFlavor::Darwin(Cc::Yes, Lld::No),
-            &["-arch", arch.target_name()],
+            &["-arch", arch.ld_arch()],
         );
     } else {
         add_link_args_iter(
             &mut args,
             LinkerFlavor::Darwin(Cc::Yes, Lld::No),
-            ["-target".into(), mac_catalyst_llvm_target(arch).into()].into_iter(),
+            ["-target".into(), llvm_target(os, arch, abi)].into_iter(),
         );
     }
 
     args
 }
 
-pub(crate) fn opts(os: &'static str, arch: Arch, abi: TargetAbi) -> TargetOptions {
-    TargetOptions {
+/// Get the base target options, LLVM target and `target_arch` from the three
+/// things that uniquely identify Rust's Apple targets: The OS, the
+/// architecture, and the ABI.
+pub(crate) fn base(
+    os: &'static str,
+    arch: Arch,
+    abi: TargetAbi,
+) -> (TargetOptions, StaticCow<str>, StaticCow<str>) {
+    let opts = TargetOptions {
         abi: abi.target_abi().into(),
         os: os.into(),
         cpu: arch.target_cpu(abi).into(),
@@ -193,10 +245,11 @@ pub(crate) fn opts(os: &'static str, arch: Arch, abi: TargetAbi) -> TargetOption
         link_env: Cow::Borrowed(&[(Cow::Borrowed("ZERO_AR_DATE"), Cow::Borrowed("1"))]),
 
         ..Default::default()
-    }
+    };
+    (opts, llvm_target(os, arch, abi), arch.target_arch())
 }
 
-pub fn sdk_version(platform: u32) -> Option<(u32, u32)> {
+pub fn sdk_version(platform: u32) -> Option<(u16, u8)> {
     // NOTE: These values are from an arbitrary point in time but shouldn't make it into the final
     // binary since the final link command will have the current SDK version passed to it.
     match platform {
@@ -230,58 +283,108 @@ pub fn platform(target: &Target) -> Option<u32> {
     })
 }
 
-pub fn deployment_target(target: &Target) -> Option<(u32, u32)> {
-    let (major, minor) = match &*target.os {
-        "macos" => {
-            // This does not need to be specific. It just needs to handle x86 vs M1.
-            let arch = match target.arch.as_ref() {
-                "x86" | "x86_64" => X86_64,
-                "arm64e" => Arm64e,
-                _ => Arm64,
-            };
-            macos_deployment_target(arch)
-        }
-        "ios" => {
-            let arch = match target.arch.as_ref() {
-                "arm64e" => Arm64e,
-                _ => Arm64,
-            };
-            ios_deployment_target(arch, &target.abi)
-        }
-        "watchos" => watchos_deployment_target(),
-        "tvos" => tvos_deployment_target(),
-        "visionos" => visionos_deployment_target(),
-        _ => return None,
+/// Hack for calling `deployment_target` outside of this module.
+pub fn deployment_target_for_target(target: &Target) -> (u16, u8, u8) {
+    let arch = if target.llvm_target.starts_with("arm64e") {
+        Arch::Arm64e
+    } else if target.arch == "aarch64" {
+        Arch::Arm64
+    } else {
+        // Dummy architecture, only used by `deployment_target` anyhow
+        Arch::X86_64
     };
-
-    Some((major, minor))
+    let abi = match &*target.abi {
+        "macabi" => TargetAbi::MacCatalyst,
+        "sim" => TargetAbi::Simulator,
+        "" => TargetAbi::Normal,
+        abi => unreachable!("invalid abi '{abi}' for Apple target"),
+    };
+    deployment_target(&target.os, arch, abi)
 }
 
-fn from_set_deployment_target(var_name: &str) -> Option<(u32, u32)> {
-    let deployment_target = env::var(var_name).ok()?;
-    let (unparsed_major, unparsed_minor) = deployment_target.split_once('.')?;
-    let (major, minor) = (unparsed_major.parse().ok()?, unparsed_minor.parse().ok()?);
+/// Get the deployment target based on the standard environment variables, or
+/// fall back to a sane default.
+fn deployment_target(os: &str, arch: Arch, abi: TargetAbi) -> (u16, u8, u8) {
+    // When bumping a version in here, remember to update the platform-support
+    // docs too.
+    //
+    // NOTE: If you are looking for the default deployment target, prefer
+    // `rustc --print deployment-target`, as the default here may change in
+    // future `rustc` versions.
+
+    // Minimum operating system versions currently supported by `rustc`.
+    let os_min = match os {
+        "macos" => (10, 12, 0),
+        "ios" => (10, 0, 0),
+        "tvos" => (10, 0, 0),
+        "watchos" => (5, 0, 0),
+        "visionos" => (1, 0, 0),
+        _ => unreachable!("tried to get deployment target for non-Apple platform"),
+    };
 
-    Some((major, minor))
-}
+    // On certain targets it makes sense to raise the minimum OS version.
+    let min = match (os, arch, abi) {
+        // Use 11.0 on Aarch64 as that's the earliest version with M1 support.
+        ("macos", Arch::Arm64 | Arch::Arm64e, _) => (11, 0, 0),
+        ("ios", Arch::Arm64e, _) => (14, 0, 0),
+        // Mac Catalyst defaults to 13.1 in Clang.
+        ("ios", _, TargetAbi::MacCatalyst) => (13, 1, 0),
+        _ => os_min,
+    };
 
-fn macos_default_deployment_target(arch: Arch) -> (u32, u32) {
-    match arch {
-        Arm64 | Arm64e => (11, 0),
-        _ => (10, 12),
-    }
-}
+    // The environment variable used to fetch the deployment target.
+    let env_var = match os {
+        "macos" => "MACOSX_DEPLOYMENT_TARGET",
+        "ios" => "IPHONEOS_DEPLOYMENT_TARGET",
+        "watchos" => "WATCHOS_DEPLOYMENT_TARGET",
+        "tvos" => "TVOS_DEPLOYMENT_TARGET",
+        "visionos" => "XROS_DEPLOYMENT_TARGET",
+        _ => unreachable!("tried to get deployment target env var for non-Apple platform"),
+    };
 
-fn macos_deployment_target(arch: Arch) -> (u32, u32) {
-    // If you are looking for the default deployment target, prefer `rustc --print deployment-target`.
-    // Note: If bumping this version, remember to update it in the rustc/platform-support docs.
-    from_set_deployment_target("MACOSX_DEPLOYMENT_TARGET")
-        .unwrap_or_else(|| macos_default_deployment_target(arch))
+    if let Ok(deployment_target) = env::var(env_var) {
+        match parse_version(&deployment_target) {
+            // It is common that the deployment target is set too low, e.g. on
+            // macOS Aarch64 to also target older x86_64, the user may set a
+            // lower deployment target than supported.
+            //
+            // To avoid such issues, we silently raise the deployment target
+            // here.
+            // FIXME: We want to show a warning when `version < os_min`.
+            Ok(version) => version.max(min),
+            // FIXME: Report erroneous environment variable to user.
+            Err(_) => min,
+        }
+    } else {
+        min
+    }
 }
 
-pub(crate) fn macos_llvm_target(arch: Arch) -> String {
-    let (major, minor) = macos_deployment_target(arch);
-    format!("{}-apple-macosx{}.{}.0", arch.target_name(), major, minor)
+/// Generate the target triple that we need to pass to LLVM and/or Clang.
+fn llvm_target(os: &str, arch: Arch, abi: TargetAbi) -> StaticCow<str> {
+    // The target triple depends on the deployment target, and is required to
+    // enable features such as cross-language LTO, and for picking the right
+    // Mach-O commands.
+    //
+    // Certain optimizations also depend on the deployment target.
+    let (major, minor, patch) = deployment_target(os, arch, abi);
+    let arch = arch.target_name();
+    // Convert to the "canonical" OS name used by LLVM:
+    // https://github.com/llvm/llvm-project/blob/llvmorg-18.1.8/llvm/lib/TargetParser/Triple.cpp#L236-L282
+    let os = match os {
+        "macos" => "macosx",
+        "ios" => "ios",
+        "watchos" => "watchos",
+        "tvos" => "tvos",
+        "visionos" => "xros",
+        _ => unreachable!("tried to get LLVM target OS for non-Apple platform"),
+    };
+    let environment = match abi {
+        TargetAbi::Normal => "",
+        TargetAbi::MacCatalyst => "-macabi",
+        TargetAbi::Simulator => "-simulator",
+    };
+    format!("{arch}-apple-{os}{major}.{minor}.{patch}{environment}").into()
 }
 
 fn link_env_remove(os: &'static str) -> StaticCow<[StaticCow<str>]> {
@@ -321,83 +424,19 @@ fn link_env_remove(os: &'static str) -> StaticCow<[StaticCow<str>]> {
     }
 }
 
-fn ios_deployment_target(arch: Arch, abi: &str) -> (u32, u32) {
-    // If you are looking for the default deployment target, prefer `rustc --print deployment-target`.
-    // Note: If bumping this version, remember to update it in the rustc/platform-support docs.
-    let (major, minor) = match (arch, abi) {
-        (Arm64e, _) => (14, 0),
-        // Mac Catalyst defaults to 13.1 in Clang.
-        (_, "macabi") => (13, 1),
-        _ => (10, 0),
-    };
-    from_set_deployment_target("IPHONEOS_DEPLOYMENT_TARGET").unwrap_or((major, minor))
-}
-
-pub(crate) fn ios_llvm_target(arch: Arch) -> String {
-    // Modern iOS tooling extracts information about deployment target
-    // from LC_BUILD_VERSION. This load command will only be emitted when
-    // we build with a version specific `llvm_target`, with the version
-    // set high enough. Luckily one LC_BUILD_VERSION is enough, for Xcode
-    // to pick it up (since std and core are still built with the fallback
-    // of version 7.0 and hence emit the old LC_IPHONE_MIN_VERSION).
-    let (major, minor) = ios_deployment_target(arch, "");
-    format!("{}-apple-ios{}.{}.0", arch.target_name(), major, minor)
-}
-
-pub(crate) fn mac_catalyst_llvm_target(arch: Arch) -> String {
-    let (major, minor) = ios_deployment_target(arch, "macabi");
-    format!("{}-apple-ios{}.{}.0-macabi", arch.target_name(), major, minor)
-}
-
-pub(crate) fn ios_sim_llvm_target(arch: Arch) -> String {
-    let (major, minor) = ios_deployment_target(arch, "sim");
-    format!("{}-apple-ios{}.{}.0-simulator", arch.target_name(), major, minor)
-}
-
-fn tvos_deployment_target() -> (u32, u32) {
-    // If you are looking for the default deployment target, prefer `rustc --print deployment-target`.
-    // Note: If bumping this version, remember to update it in the rustc platform-support docs.
-    from_set_deployment_target("TVOS_DEPLOYMENT_TARGET").unwrap_or((10, 0))
-}
-
-pub(crate) fn tvos_llvm_target(arch: Arch) -> String {
-    let (major, minor) = tvos_deployment_target();
-    format!("{}-apple-tvos{}.{}.0", arch.target_name(), major, minor)
-}
-
-pub(crate) fn tvos_sim_llvm_target(arch: Arch) -> String {
-    let (major, minor) = tvos_deployment_target();
-    format!("{}-apple-tvos{}.{}.0-simulator", arch.target_name(), major, minor)
-}
-
-fn watchos_deployment_target() -> (u32, u32) {
-    // If you are looking for the default deployment target, prefer `rustc --print deployment-target`.
-    // Note: If bumping this version, remember to update it in the rustc platform-support docs.
-    from_set_deployment_target("WATCHOS_DEPLOYMENT_TARGET").unwrap_or((5, 0))
-}
-
-pub(crate) fn watchos_llvm_target(arch: Arch) -> String {
-    let (major, minor) = watchos_deployment_target();
-    format!("{}-apple-watchos{}.{}.0", arch.target_name(), major, minor)
-}
-
-pub(crate) fn watchos_sim_llvm_target(arch: Arch) -> String {
-    let (major, minor) = watchos_deployment_target();
-    format!("{}-apple-watchos{}.{}.0-simulator", arch.target_name(), major, minor)
-}
-
-fn visionos_deployment_target() -> (u32, u32) {
-    // If you are looking for the default deployment target, prefer `rustc --print deployment-target`.
-    // Note: If bumping this version, remember to update it in the rustc platform-support docs.
-    from_set_deployment_target("XROS_DEPLOYMENT_TARGET").unwrap_or((1, 0))
-}
-
-pub(crate) fn visionos_llvm_target(arch: Arch) -> String {
-    let (major, minor) = visionos_deployment_target();
-    format!("{}-apple-visionos{}.{}.0", arch.target_name(), major, minor)
-}
-
-pub(crate) fn visionos_sim_llvm_target(arch: Arch) -> String {
-    let (major, minor) = visionos_deployment_target();
-    format!("{}-apple-visionos{}.{}.0-simulator", arch.target_name(), major, minor)
+/// Parse an OS version triple (SDK version or deployment target).
+///
+/// The size of the returned numbers here are limited by Mach-O's
+/// `LC_BUILD_VERSION`.
+fn parse_version(version: &str) -> Result<(u16, u8, u8), ParseIntError> {
+    if let Some((major, minor)) = version.split_once('.') {
+        let major = major.parse()?;
+        if let Some((minor, patch)) = minor.split_once('.') {
+            Ok((major, minor.parse()?, patch.parse()?))
+        } else {
+            Ok((major, minor.parse()?, 0))
+        }
+    } else {
+        Ok((version.parse()?, 0, 0))
+    }
 }
diff --git a/compiler/rustc_target/src/spec/base/apple/tests.rs b/compiler/rustc_target/src/spec/base/apple/tests.rs
index 7a985ad4dc0..d55b44e3c9e 100644
--- a/compiler/rustc_target/src/spec/base/apple/tests.rs
+++ b/compiler/rustc_target/src/spec/base/apple/tests.rs
@@ -1,3 +1,4 @@
+use super::parse_version;
 use crate::spec::targets::{
     aarch64_apple_darwin, aarch64_apple_ios_sim, aarch64_apple_visionos_sim,
     aarch64_apple_watchos_sim, i686_apple_darwin, x86_64_apple_darwin, x86_64_apple_ios,
@@ -42,3 +43,11 @@ fn macos_link_environment_unmodified() {
         );
     }
 }
+
+#[test]
+fn test_parse_version() {
+    assert_eq!(parse_version("10"), Ok((10, 0, 0)));
+    assert_eq!(parse_version("10.12"), Ok((10, 12, 0)));
+    assert_eq!(parse_version("10.12.6"), Ok((10, 12, 6)));
+    assert_eq!(parse_version("9999.99.99"), Ok((9999, 99, 99)));
+}
diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
index 6abd8a0d6b7..c3e7f74c564 100644
--- a/compiler/rustc_target/src/spec/mod.rs
+++ b/compiler/rustc_target/src/spec/mod.rs
@@ -60,8 +60,8 @@ pub mod crt_objects;
 
 mod base;
 pub use base::apple::{
-    deployment_target as current_apple_deployment_target, platform as current_apple_platform,
-    sdk_version as current_apple_sdk_version,
+    deployment_target_for_target as current_apple_deployment_target,
+    platform as current_apple_platform, sdk_version as current_apple_sdk_version,
 };
 pub use base::avr_gnu::ef_avr_arch;
 
@@ -1645,6 +1645,7 @@ supported_targets! {
     ("x86_64-unknown-haiku", x86_64_unknown_haiku),
 
     ("i686-unknown-hurd-gnu", i686_unknown_hurd_gnu),
+    ("x86_64-unknown-hurd-gnu", x86_64_unknown_hurd_gnu),
 
     ("aarch64-apple-darwin", aarch64_apple_darwin),
     ("arm64e-apple-darwin", arm64e_apple_darwin),
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_darwin.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_darwin.rs
index 912392c9fef..1061633be17 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_darwin.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_darwin.rs
@@ -1,20 +1,10 @@
-use crate::spec::base::apple::{macos_llvm_target, opts, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{FramePointer, SanitizerSet, Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    let arch = Arch::Arm64;
-    let mut base = opts("macos", arch, TargetAbi::Normal);
-    base.cpu = "apple-m1".into();
-    base.max_atomic_width = Some(128);
-
-    // FIXME: The leak sanitizer currently fails the tests, see #88132.
-    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::CFI | SanitizerSet::THREAD;
-
+    let (opts, llvm_target, arch) = base("macos", Arch::Arm64, TargetAbi::Normal);
     Target {
-        // Clang automatically chooses a more specific target based on
-        // MACOSX_DEPLOYMENT_TARGET. To enable cross-language LTO to work
-        // correctly, we do too.
-        llvm_target: macos_llvm_target(arch).into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("ARM64 macOS (11.0+, Big Sur+)".into()),
             tier: Some(1),
@@ -23,11 +13,15 @@ pub(crate) fn target() -> Target {
         },
         pointer_width: 64,
         data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(),
-        arch: arch.target_arch(),
+        arch,
         options: TargetOptions {
             mcount: "\u{1}mcount".into(),
             frame_pointer: FramePointer::NonLeaf,
-            ..base
+            cpu: "apple-m1".into(),
+            max_atomic_width: Some(128),
+            // FIXME: The leak sanitizer currently fails the tests, see #88132.
+            supported_sanitizers: SanitizerSet::ADDRESS | SanitizerSet::CFI | SanitizerSet::THREAD,
+            ..opts
         },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios.rs
index 7f3a86f3256..caeb9a121e4 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios.rs
@@ -1,17 +1,10 @@
-use crate::spec::base::apple::{ios_llvm_target, opts, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{FramePointer, SanitizerSet, Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    let arch = Arch::Arm64;
-    let mut base = opts("ios", arch, TargetAbi::Normal);
-    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::THREAD;
-
+    let (opts, llvm_target, arch) = base("ios", Arch::Arm64, TargetAbi::Normal);
     Target {
-        // Clang automatically chooses a more specific target based on
-        // IPHONEOS_DEPLOYMENT_TARGET.
-        // This is required for the target to pick the right
-        // MACH-O commands, so we do too.
-        llvm_target: ios_llvm_target(arch).into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("ARM64 iOS".into()),
             tier: Some(2),
@@ -20,12 +13,13 @@ pub(crate) fn target() -> Target {
         },
         pointer_width: 64,
         data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(),
-        arch: arch.target_arch(),
+        arch,
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a7".into(),
             max_atomic_width: Some(128),
             frame_pointer: FramePointer::NonLeaf,
-            ..base
+            supported_sanitizers: SanitizerSet::ADDRESS | SanitizerSet::THREAD,
+            ..opts
         },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_macabi.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_macabi.rs
index e60eeda4295..eee9eca3bcb 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_macabi.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_macabi.rs
@@ -1,13 +1,10 @@
-use crate::spec::base::apple::{mac_catalyst_llvm_target, opts, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{FramePointer, SanitizerSet, Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    let arch = Arch::Arm64;
-    let mut base = opts("ios", arch, TargetAbi::MacCatalyst);
-    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::LEAK | SanitizerSet::THREAD;
-
+    let (opts, llvm_target, arch) = base("ios", Arch::Arm64, TargetAbi::MacCatalyst);
     Target {
-        llvm_target: mac_catalyst_llvm_target(arch).into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("Apple Catalyst on ARM64".into()),
             tier: Some(2),
@@ -16,12 +13,13 @@ pub(crate) fn target() -> Target {
         },
         pointer_width: 64,
         data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(),
-        arch: arch.target_arch(),
+        arch,
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a12".into(),
             max_atomic_width: Some(128),
             frame_pointer: FramePointer::NonLeaf,
-            ..base
+            supported_sanitizers: SanitizerSet::ADDRESS | SanitizerSet::LEAK | SanitizerSet::THREAD,
+            ..opts
         },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_sim.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_sim.rs
index d605e22701a..ee58038301d 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_sim.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_sim.rs
@@ -1,17 +1,10 @@
-use crate::spec::base::apple::{ios_sim_llvm_target, opts, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{FramePointer, SanitizerSet, Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    let arch = Arch::Arm64;
-    let mut base = opts("ios", arch, TargetAbi::Simulator);
-    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::THREAD;
-
+    let (opts, llvm_target, arch) = base("ios", Arch::Arm64, TargetAbi::Simulator);
     Target {
-        // Clang automatically chooses a more specific target based on
-        // IPHONEOS_DEPLOYMENT_TARGET.
-        // This is required for the simulator target to pick the right
-        // MACH-O commands, so we do too.
-        llvm_target: ios_sim_llvm_target(arch).into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("Apple iOS Simulator on ARM64".into()),
             tier: Some(2),
@@ -20,12 +13,13 @@ pub(crate) fn target() -> Target {
         },
         pointer_width: 64,
         data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(),
-        arch: arch.target_arch(),
+        arch,
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a7".into(),
             max_atomic_width: Some(128),
             frame_pointer: FramePointer::NonLeaf,
-            ..base
+            supported_sanitizers: SanitizerSet::ADDRESS | SanitizerSet::THREAD,
+            ..opts
         },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos.rs
index fc774ab94c8..baca863d442 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos.rs
@@ -1,10 +1,10 @@
-use crate::spec::base::apple::{opts, tvos_llvm_target, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{FramePointer, Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    let arch = Arch::Arm64;
+    let (opts, llvm_target, arch) = base("tvos", Arch::Arm64, TargetAbi::Normal);
     Target {
-        llvm_target: tvos_llvm_target(arch).into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("ARM64 tvOS".into()),
             tier: Some(3),
@@ -13,12 +13,12 @@ pub(crate) fn target() -> Target {
         },
         pointer_width: 64,
         data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(),
-        arch: arch.target_arch(),
+        arch,
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a7".into(),
             max_atomic_width: Some(128),
             frame_pointer: FramePointer::NonLeaf,
-            ..opts("tvos", arch, TargetAbi::Normal)
+            ..opts
         },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos_sim.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos_sim.rs
index 8e8bb1efc9d..1a48f8c5acf 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos_sim.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos_sim.rs
@@ -1,10 +1,10 @@
-use crate::spec::base::apple::{opts, tvos_sim_llvm_target, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{FramePointer, Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    let arch = Arch::Arm64;
+    let (opts, llvm_target, arch) = base("tvos", Arch::Arm64, TargetAbi::Simulator);
     Target {
-        llvm_target: tvos_sim_llvm_target(arch).into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("ARM64 tvOS Simulator".into()),
             tier: Some(3),
@@ -13,12 +13,12 @@ pub(crate) fn target() -> Target {
         },
         pointer_width: 64,
         data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(),
-        arch: arch.target_arch(),
+        arch,
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a7".into(),
             max_atomic_width: Some(128),
             frame_pointer: FramePointer::NonLeaf,
-            ..opts("tvos", arch, TargetAbi::Simulator)
+            ..opts
         },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos.rs
index 16c7d72daed..1424126134d 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos.rs
@@ -1,13 +1,10 @@
-use crate::spec::base::apple::{opts, visionos_llvm_target, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{FramePointer, SanitizerSet, Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    let arch = Arch::Arm64;
-    let mut base = opts("visionos", arch, TargetAbi::Normal);
-    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::THREAD;
-
+    let (opts, llvm_target, arch) = base("visionos", Arch::Arm64, TargetAbi::Normal);
     Target {
-        llvm_target: visionos_llvm_target(arch).into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("ARM64 Apple visionOS".into()),
             tier: Some(3),
@@ -16,12 +13,13 @@ pub(crate) fn target() -> Target {
         },
         pointer_width: 64,
         data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(),
-        arch: arch.target_arch(),
+        arch,
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a16".into(),
             max_atomic_width: Some(128),
             frame_pointer: FramePointer::NonLeaf,
-            ..base
+            supported_sanitizers: SanitizerSet::ADDRESS | SanitizerSet::THREAD,
+            ..opts
         },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos_sim.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos_sim.rs
index 42973e0268e..d7226e02ecb 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos_sim.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_visionos_sim.rs
@@ -1,13 +1,10 @@
-use crate::spec::base::apple::{opts, visionos_sim_llvm_target, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{FramePointer, SanitizerSet, Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    let arch = Arch::Arm64;
-    let mut base = opts("visionos", arch, TargetAbi::Simulator);
-    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::THREAD;
-
+    let (opts, llvm_target, arch) = base("visionos", Arch::Arm64, TargetAbi::Simulator);
     Target {
-        llvm_target: visionos_sim_llvm_target(arch).into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("ARM64 Apple visionOS simulator".into()),
             tier: Some(3),
@@ -16,12 +13,13 @@ pub(crate) fn target() -> Target {
         },
         pointer_width: 64,
         data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(),
-        arch: arch.target_arch(),
+        arch,
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a16".into(),
             max_atomic_width: Some(128),
             frame_pointer: FramePointer::NonLeaf,
-            ..base
+            supported_sanitizers: SanitizerSet::ADDRESS | SanitizerSet::THREAD,
+            ..opts
         },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos.rs
index 03fc2518d1d..1940a568b39 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos.rs
@@ -1,10 +1,10 @@
-use crate::spec::base::apple::{opts, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    let base = opts("watchos", Arch::Arm64, TargetAbi::Normal);
+    let (opts, llvm_target, arch) = base("watchos", Arch::Arm64, TargetAbi::Normal);
     Target {
-        llvm_target: "aarch64-apple-watchos".into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("ARM64 Apple WatchOS".into()),
             tier: Some(3),
@@ -13,13 +13,13 @@ pub(crate) fn target() -> Target {
         },
         pointer_width: 64,
         data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(),
-        arch: "aarch64".into(),
+        arch,
         options: TargetOptions {
             features: "+v8a,+neon,+fp-armv8,+apple-a7".into(),
             max_atomic_width: Some(128),
             dynamic_linking: false,
             position_independent_executables: true,
-            ..base
+            ..opts
         },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos_sim.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos_sim.rs
index 13e2b961794..6beef11c504 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos_sim.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos_sim.rs
@@ -1,14 +1,10 @@
-use crate::spec::base::apple::{opts, watchos_sim_llvm_target, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{FramePointer, Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    let arch = Arch::Arm64;
+    let (opts, llvm_target, arch) = base("watchos", Arch::Arm64, TargetAbi::Simulator);
     Target {
-        // Clang automatically chooses a more specific target based on
-        // WATCHOS_DEPLOYMENT_TARGET.
-        // This is required for the simulator target to pick the right
-        // MACH-O commands, so we do too.
-        llvm_target: watchos_sim_llvm_target(arch).into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("ARM64 Apple WatchOS Simulator".into()),
             tier: Some(3),
@@ -17,12 +13,12 @@ pub(crate) fn target() -> Target {
         },
         pointer_width: 64,
         data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(),
-        arch: arch.target_arch(),
+        arch,
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a7".into(),
             max_atomic_width: Some(128),
             frame_pointer: FramePointer::NonLeaf,
-            ..opts("watchos", arch, TargetAbi::Simulator)
+            ..opts
         },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/arm64_32_apple_watchos.rs b/compiler/rustc_target/src/spec/targets/arm64_32_apple_watchos.rs
index f292ee7f563..e9f4d9330d5 100644
--- a/compiler/rustc_target/src/spec/targets/arm64_32_apple_watchos.rs
+++ b/compiler/rustc_target/src/spec/targets/arm64_32_apple_watchos.rs
@@ -1,11 +1,10 @@
-use crate::spec::base::apple::{opts, watchos_llvm_target, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    let arch = Arch::Arm64_32;
-    let base = opts("watchos", arch, TargetAbi::Normal);
+    let (opts, llvm_target, arch) = base("watchos", Arch::Arm64_32, TargetAbi::Normal);
     Target {
-        llvm_target: watchos_llvm_target(arch).into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("Arm Apple WatchOS 64-bit with 32-bit pointers".into()),
             tier: Some(3),
@@ -14,13 +13,13 @@ pub(crate) fn target() -> Target {
         },
         pointer_width: 32,
         data_layout: "e-m:o-p:32:32-i64:64-i128:128-n32:64-S128-Fn32".into(),
-        arch: "aarch64".into(),
+        arch,
         options: TargetOptions {
             features: "+v8a,+neon,+fp-armv8,+apple-a7".into(),
             max_atomic_width: Some(128),
             dynamic_linking: false,
             position_independent_executables: true,
-            ..base
+            ..opts
         },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/arm64e_apple_darwin.rs b/compiler/rustc_target/src/spec/targets/arm64e_apple_darwin.rs
index e86b8358a9c..4f7c945b9fe 100644
--- a/compiler/rustc_target/src/spec/targets/arm64e_apple_darwin.rs
+++ b/compiler/rustc_target/src/spec/targets/arm64e_apple_darwin.rs
@@ -1,20 +1,10 @@
-use crate::spec::base::apple::{macos_llvm_target, opts, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{FramePointer, SanitizerSet, Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    let arch = Arch::Arm64e;
-    let mut base = opts("macos", arch, TargetAbi::Normal);
-    base.cpu = "apple-m1".into();
-    base.max_atomic_width = Some(128);
-
-    // FIXME: The leak sanitizer currently fails the tests, see #88132.
-    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::CFI | SanitizerSet::THREAD;
-
+    let (opts, llvm_target, arch) = base("macos", Arch::Arm64e, TargetAbi::Normal);
     Target {
-        // Clang automatically chooses a more specific target based on
-        // MACOSX_DEPLOYMENT_TARGET. To enable cross-language LTO to work
-        // correctly, we do too.
-        llvm_target: macos_llvm_target(arch).into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("ARM64e Apple Darwin".into()),
             tier: Some(3),
@@ -23,11 +13,15 @@ pub(crate) fn target() -> Target {
         },
         pointer_width: 64,
         data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(),
-        arch: arch.target_arch(),
+        arch,
         options: TargetOptions {
             mcount: "\u{1}mcount".into(),
             frame_pointer: FramePointer::NonLeaf,
-            ..base
+            cpu: "apple-m1".into(),
+            max_atomic_width: Some(128),
+            // FIXME: The leak sanitizer currently fails the tests, see #88132.
+            supported_sanitizers: SanitizerSet::ADDRESS | SanitizerSet::CFI | SanitizerSet::THREAD,
+            ..opts
         },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/arm64e_apple_ios.rs b/compiler/rustc_target/src/spec/targets/arm64e_apple_ios.rs
index fcb850f44bc..8f192fe2886 100644
--- a/compiler/rustc_target/src/spec/targets/arm64e_apple_ios.rs
+++ b/compiler/rustc_target/src/spec/targets/arm64e_apple_ios.rs
@@ -1,17 +1,10 @@
-use crate::spec::base::apple::{ios_llvm_target, opts, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{FramePointer, SanitizerSet, Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    let arch = Arch::Arm64e;
-    let mut base = opts("ios", arch, TargetAbi::Normal);
-    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::THREAD;
-
+    let (opts, llvm_target, arch) = base("ios", Arch::Arm64e, TargetAbi::Normal);
     Target {
-        // Clang automatically chooses a more specific target based on
-        // IPHONEOS_DEPLOYMENT_TARGET.
-        // This is required for the target to pick the right
-        // MACH-O commands, so we do too.
-        llvm_target: ios_llvm_target(arch).into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("ARM64e Apple iOS".into()),
             tier: Some(3),
@@ -20,12 +13,13 @@ pub(crate) fn target() -> Target {
         },
         pointer_width: 64,
         data_layout: "e-m:o-i64:64-i128:128-n32:64-S128-Fn32".into(),
-        arch: arch.target_arch(),
+        arch,
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a12,+v8.3a,+pauth".into(),
             max_atomic_width: Some(128),
             frame_pointer: FramePointer::NonLeaf,
-            ..base
+            supported_sanitizers: SanitizerSet::ADDRESS | SanitizerSet::THREAD,
+            ..opts
         },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/armv7k_apple_watchos.rs b/compiler/rustc_target/src/spec/targets/armv7k_apple_watchos.rs
index 35de6b35eaf..8fe27f60e0a 100644
--- a/compiler/rustc_target/src/spec/targets/armv7k_apple_watchos.rs
+++ b/compiler/rustc_target/src/spec/targets/armv7k_apple_watchos.rs
@@ -1,10 +1,10 @@
-use crate::spec::base::apple::{opts, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    let arch = Arch::Armv7k;
+    let (opts, llvm_target, arch) = base("watchos", Arch::Armv7k, TargetAbi::Normal);
     Target {
-        llvm_target: "armv7k-apple-watchos".into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("Armv7-A Apple WatchOS".into()),
             tier: Some(3),
@@ -13,13 +13,13 @@ pub(crate) fn target() -> Target {
         },
         pointer_width: 32,
         data_layout: "e-m:o-p:32:32-Fi8-i64:64-a:0:32-n32-S128".into(),
-        arch: arch.target_arch(),
+        arch,
         options: TargetOptions {
             features: "+v7,+vfp4,+neon".into(),
             max_atomic_width: Some(64),
             dynamic_linking: false,
             position_independent_executables: true,
-            ..opts("watchos", arch, TargetAbi::Normal)
+            ..opts
         },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/armv7s_apple_ios.rs b/compiler/rustc_target/src/spec/targets/armv7s_apple_ios.rs
index 41eaa79d3eb..deee6985f1a 100644
--- a/compiler/rustc_target/src/spec/targets/armv7s_apple_ios.rs
+++ b/compiler/rustc_target/src/spec/targets/armv7s_apple_ios.rs
@@ -1,10 +1,10 @@
-use crate::spec::base::apple::{ios_llvm_target, opts, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    let arch = Arch::Armv7s;
+    let (opts, llvm_target, arch) = base("ios", Arch::Armv7s, TargetAbi::Normal);
     Target {
-        llvm_target: ios_llvm_target(arch).into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("Armv7-A Apple-A6 Apple iOS".into()),
             tier: Some(3),
@@ -13,11 +13,11 @@ pub(crate) fn target() -> Target {
         },
         pointer_width: 32,
         data_layout: "e-m:o-p:32:32-Fi8-f64:32:64-v64:32:64-v128:32:128-a:0:32-n32-S32".into(),
-        arch: arch.target_arch(),
+        arch,
         options: TargetOptions {
             features: "+v7,+vfp4,+neon".into(),
             max_atomic_width: Some(64),
-            ..opts("ios", arch, TargetAbi::Normal)
+            ..opts
         },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/i386_apple_ios.rs b/compiler/rustc_target/src/spec/targets/i386_apple_ios.rs
index 263393fd536..dc14cb3ec76 100644
--- a/compiler/rustc_target/src/spec/targets/i386_apple_ios.rs
+++ b/compiler/rustc_target/src/spec/targets/i386_apple_ios.rs
@@ -1,17 +1,12 @@
-use crate::spec::base::apple::{ios_sim_llvm_target, opts, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    let arch = Arch::I386;
     // i386-apple-ios is a simulator target, even though it isn't declared
     // that way in the target name like the other ones...
-    let abi = TargetAbi::Simulator;
+    let (opts, llvm_target, arch) = base("ios", Arch::I386, TargetAbi::Simulator);
     Target {
-        // Clang automatically chooses a more specific target based on
-        // IPHONEOS_DEPLOYMENT_TARGET.
-        // This is required for the target to pick the right
-        // MACH-O commands, so we do too.
-        llvm_target: ios_sim_llvm_target(arch).into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("32-bit x86 iOS".into()),
             tier: Some(3),
@@ -22,7 +17,7 @@ pub(crate) fn target() -> Target {
         data_layout: "e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
             i128:128-f64:32:64-f80:128-n8:16:32-S128"
             .into(),
-        arch: arch.target_arch(),
-        options: TargetOptions { max_atomic_width: Some(64), ..opts("ios", arch, abi) },
+        arch,
+        options: TargetOptions { max_atomic_width: Some(64), ..opts },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/i686_apple_darwin.rs b/compiler/rustc_target/src/spec/targets/i686_apple_darwin.rs
index f173e8b7e4a..67afe35bee4 100644
--- a/compiler/rustc_target/src/spec/targets/i686_apple_darwin.rs
+++ b/compiler/rustc_target/src/spec/targets/i686_apple_darwin.rs
@@ -1,21 +1,12 @@
-use crate::spec::base::apple::{macos_llvm_target, opts, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{Cc, FramePointer, LinkerFlavor, Lld, Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    // ld64 only understands i386 and not i686
-    let arch = Arch::I386;
-    let mut base = opts("macos", arch, TargetAbi::Normal);
-    base.max_atomic_width = Some(64);
-    base.add_pre_link_args(LinkerFlavor::Darwin(Cc::Yes, Lld::No), &["-m32"]);
-    base.frame_pointer = FramePointer::Always;
+    let (mut opts, llvm_target, arch) = base("macos", Arch::I686, TargetAbi::Normal);
+    opts.add_pre_link_args(LinkerFlavor::Darwin(Cc::Yes, Lld::No), &["-m32"]);
 
     Target {
-        // Clang automatically chooses a more specific target based on
-        // MACOSX_DEPLOYMENT_TARGET. To enable cross-language LTO to work
-        // correctly, we do too.
-        //
-        // While ld64 doesn't understand i686, LLVM does.
-        llvm_target: macos_llvm_target(Arch::I686).into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("32-bit macOS (10.12+, Sierra+)".into()),
             tier: Some(3),
@@ -26,7 +17,12 @@ pub(crate) fn target() -> Target {
         data_layout: "e-m:o-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
             i128:128-f64:32:64-f80:128-n8:16:32-S128"
             .into(),
-        arch: arch.target_arch(),
-        options: TargetOptions { mcount: "\u{1}mcount".into(), ..base },
+        arch,
+        options: TargetOptions {
+            mcount: "\u{1}mcount".into(),
+            max_atomic_width: Some(64),
+            frame_pointer: FramePointer::Always,
+            ..opts
+        },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/wasm32_wasip2.rs b/compiler/rustc_target/src/spec/targets/wasm32_wasip2.rs
index 40f026ca938..876aa7073f8 100644
--- a/compiler/rustc_target/src/spec/targets/wasm32_wasip2.rs
+++ b/compiler/rustc_target/src/spec/targets/wasm32_wasip2.rs
@@ -1,6 +1,6 @@
 //! The `wasm32-wasip2` target is the next evolution of the
 //! wasm32-wasi target. While the wasi specification is still under
-//! active development, the {review 2 iteration is considered an "island
+//! active development, the preview 2 iteration is considered an "island
 //! of stability" that should allow users to rely on it indefinitely.
 //!
 //! The `wasi` target is a proposal to define a standardized set of WebAssembly
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_apple_darwin.rs b/compiler/rustc_target/src/spec/targets/x86_64_apple_darwin.rs
index 7bf18026735..e7f14aa9209 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_apple_darwin.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_apple_darwin.rs
@@ -1,20 +1,11 @@
-use crate::spec::base::apple::{macos_llvm_target, opts, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{Cc, FramePointer, LinkerFlavor, Lld, SanitizerSet, Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    let arch = Arch::X86_64;
-    let mut base = opts("macos", arch, TargetAbi::Normal);
-    base.max_atomic_width = Some(128); // penryn+ supports cmpxchg16b
-    base.frame_pointer = FramePointer::Always;
-    base.add_pre_link_args(LinkerFlavor::Darwin(Cc::Yes, Lld::No), &["-m64"]);
-    base.supported_sanitizers =
-        SanitizerSet::ADDRESS | SanitizerSet::CFI | SanitizerSet::LEAK | SanitizerSet::THREAD;
-
+    let (mut opts, llvm_target, arch) = base("macos", Arch::X86_64, TargetAbi::Normal);
+    opts.add_pre_link_args(LinkerFlavor::Darwin(Cc::Yes, Lld::No), &["-m64"]);
     Target {
-        // Clang automatically chooses a more specific target based on
-        // MACOSX_DEPLOYMENT_TARGET. To enable cross-language LTO to work
-        // correctly, we do too.
-        llvm_target: macos_llvm_target(arch).into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("64-bit macOS (10.12+, Sierra+)".into()),
             tier: Some(1),
@@ -24,7 +15,16 @@ pub(crate) fn target() -> Target {
         pointer_width: 64,
         data_layout:
             "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128".into(),
-        arch: arch.target_arch(),
-        options: TargetOptions { mcount: "\u{1}mcount".into(), ..base },
+        arch,
+        options: TargetOptions {
+            mcount: "\u{1}mcount".into(),
+            max_atomic_width: Some(128), // penryn+ supports cmpxchg16b
+            frame_pointer: FramePointer::Always,
+            supported_sanitizers: SanitizerSet::ADDRESS
+                | SanitizerSet::CFI
+                | SanitizerSet::LEAK
+                | SanitizerSet::THREAD,
+            ..opts
+        },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_apple_ios.rs b/compiler/rustc_target/src/spec/targets/x86_64_apple_ios.rs
index 76a0bb1d91e..847c4f011f9 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_apple_ios.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_apple_ios.rs
@@ -1,15 +1,12 @@
-use crate::spec::base::apple::{ios_sim_llvm_target, opts, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{SanitizerSet, Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    let arch = Arch::X86_64;
     // x86_64-apple-ios is a simulator target, even though it isn't declared
     // that way in the target name like the other ones...
-    let mut base = opts("ios", arch, TargetAbi::Simulator);
-    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::THREAD;
-
+    let (opts, llvm_target, arch) = base("ios", Arch::X86_64, TargetAbi::Simulator);
     Target {
-        llvm_target: ios_sim_llvm_target(arch).into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("64-bit x86 iOS".into()),
             tier: Some(2),
@@ -19,7 +16,11 @@ pub(crate) fn target() -> Target {
         pointer_width: 64,
         data_layout:
             "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128".into(),
-        arch: arch.target_arch(),
-        options: TargetOptions { max_atomic_width: Some(128), ..base },
+        arch,
+        options: TargetOptions {
+            max_atomic_width: Some(128),
+            supported_sanitizers: SanitizerSet::ADDRESS | SanitizerSet::THREAD,
+            ..opts
+        },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_apple_ios_macabi.rs b/compiler/rustc_target/src/spec/targets/x86_64_apple_ios_macabi.rs
index f66efbbf709..042079f800b 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_apple_ios_macabi.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_apple_ios_macabi.rs
@@ -1,13 +1,10 @@
-use crate::spec::base::apple::{mac_catalyst_llvm_target, opts, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{SanitizerSet, Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    let arch = Arch::X86_64;
-    let mut base = opts("ios", arch, TargetAbi::MacCatalyst);
-    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::LEAK | SanitizerSet::THREAD;
-
+    let (opts, llvm_target, arch) = base("ios", Arch::X86_64, TargetAbi::MacCatalyst);
     Target {
-        llvm_target: mac_catalyst_llvm_target(arch).into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("Apple Catalyst on x86_64".into()),
             tier: Some(2),
@@ -17,7 +14,11 @@ pub(crate) fn target() -> Target {
         pointer_width: 64,
         data_layout:
             "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128".into(),
-        arch: arch.target_arch(),
-        options: TargetOptions { max_atomic_width: Some(128), ..base },
+        arch,
+        options: TargetOptions {
+            max_atomic_width: Some(128),
+            supported_sanitizers: SanitizerSet::ADDRESS | SanitizerSet::LEAK | SanitizerSet::THREAD,
+            ..opts
+        },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_apple_tvos.rs b/compiler/rustc_target/src/spec/targets/x86_64_apple_tvos.rs
index f9d5f6e77d9..94708609790 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_apple_tvos.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_apple_tvos.rs
@@ -1,13 +1,12 @@
-use crate::spec::base::apple::{opts, tvos_sim_llvm_target, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    let arch = Arch::X86_64;
     // x86_64-apple-tvos is a simulator target, even though it isn't declared
     // that way in the target name like the other ones...
-    let abi = TargetAbi::Simulator;
+    let (opts, llvm_target, arch) = base("tvos", Arch::X86_64, TargetAbi::Simulator);
     Target {
-        llvm_target: tvos_sim_llvm_target(arch).into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("x86 64-bit tvOS".into()),
             tier: Some(3),
@@ -17,7 +16,7 @@ pub(crate) fn target() -> Target {
         pointer_width: 64,
         data_layout:
             "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128".into(),
-        arch: arch.target_arch(),
-        options: TargetOptions { max_atomic_width: Some(128), ..opts("tvos", arch, abi) },
+        arch,
+        options: TargetOptions { max_atomic_width: Some(128), ..opts },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_apple_watchos_sim.rs b/compiler/rustc_target/src/spec/targets/x86_64_apple_watchos_sim.rs
index 1fced10c895..1dab9598860 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_apple_watchos_sim.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_apple_watchos_sim.rs
@@ -1,10 +1,10 @@
-use crate::spec::base::apple::{opts, watchos_sim_llvm_target, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    let arch = Arch::X86_64;
+    let (opts, llvm_target, arch) = base("watchos", Arch::X86_64, TargetAbi::Simulator);
     Target {
-        llvm_target: watchos_sim_llvm_target(arch).into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("x86 64-bit Apple WatchOS simulator".into()),
             tier: Some(3),
@@ -14,10 +14,7 @@ pub(crate) fn target() -> Target {
         pointer_width: 64,
         data_layout:
             "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128".into(),
-        arch: arch.target_arch(),
-        options: TargetOptions {
-            max_atomic_width: Some(128),
-            ..opts("watchos", arch, TargetAbi::Simulator)
-        },
+        arch,
+        options: TargetOptions { max_atomic_width: Some(128), ..opts },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_unknown_hurd_gnu.rs b/compiler/rustc_target/src/spec/targets/x86_64_unknown_hurd_gnu.rs
new file mode 100644
index 00000000000..6461ece5dd1
--- /dev/null
+++ b/compiler/rustc_target/src/spec/targets/x86_64_unknown_hurd_gnu.rs
@@ -0,0 +1,26 @@
+use crate::spec::{base, Cc, LinkerFlavor, Lld, StackProbeType, Target};
+
+pub(crate) fn target() -> Target {
+    let mut base = base::hurd_gnu::opts();
+    base.cpu = "x86-64".into();
+    base.plt_by_default = false;
+    base.max_atomic_width = Some(64);
+    base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m64"]);
+    base.stack_probes = StackProbeType::Inline;
+    base.supports_xray = true;
+
+    Target {
+        llvm_target: "x86_64-unknown-hurd-gnu".into(),
+        metadata: crate::spec::TargetMetadata {
+            description: Some("64-bit GNU/Hurd".into()),
+            tier: Some(3),
+            host_tools: Some(true),
+            std: Some(true),
+        },
+        pointer_width: 64,
+        data_layout:
+            "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128".into(),
+        arch: "x86_64".into(),
+        options: base,
+    }
+}
diff --git a/compiler/rustc_target/src/spec/targets/x86_64h_apple_darwin.rs b/compiler/rustc_target/src/spec/targets/x86_64h_apple_darwin.rs
index 9b9ffe4e536..f44bc660a62 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64h_apple_darwin.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64h_apple_darwin.rs
@@ -1,13 +1,12 @@
-use crate::spec::base::apple::{macos_llvm_target, opts, Arch, TargetAbi};
+use crate::spec::base::apple::{base, Arch, TargetAbi};
 use crate::spec::{Cc, FramePointer, LinkerFlavor, Lld, SanitizerSet, Target, TargetOptions};
 
 pub(crate) fn target() -> Target {
-    let arch = Arch::X86_64h;
-    let mut base = opts("macos", arch, TargetAbi::Normal);
-    base.max_atomic_width = Some(128);
-    base.frame_pointer = FramePointer::Always;
-    base.add_pre_link_args(LinkerFlavor::Darwin(Cc::Yes, Lld::No), &["-m64"]);
-    base.supported_sanitizers =
+    let (mut opts, llvm_target, arch) = base("macos", Arch::X86_64h, TargetAbi::Normal);
+    opts.max_atomic_width = Some(128);
+    opts.frame_pointer = FramePointer::Always;
+    opts.add_pre_link_args(LinkerFlavor::Darwin(Cc::Yes, Lld::No), &["-m64"]);
+    opts.supported_sanitizers =
         SanitizerSet::ADDRESS | SanitizerSet::CFI | SanitizerSet::LEAK | SanitizerSet::THREAD;
 
     // x86_64h is core2-avx without a few of the features which would otherwise
@@ -20,19 +19,16 @@ pub(crate) fn target() -> Target {
     // It would be nice if this were not the case, but fixing it seems tricky
     // (and given that the main use-case for this target is for use in universal
     // binaries, probably not that important).
-    base.features = "-rdrnd,-aes,-pclmul,-rtm,-fsgsbase".into();
+    opts.features = "-rdrnd,-aes,-pclmul,-rtm,-fsgsbase".into();
     // Double-check that the `cpu` is what we expect (if it's not the list above
     // may need updating).
     assert_eq!(
-        base.cpu, "core-avx2",
+        opts.cpu, "core-avx2",
         "you need to adjust the feature list in x86_64h-apple-darwin if you change this",
     );
 
     Target {
-        // Clang automatically chooses a more specific target based on
-        // MACOSX_DEPLOYMENT_TARGET. To enable cross-language LTO to work
-        // correctly, we do too.
-        llvm_target: macos_llvm_target(arch).into(),
+        llvm_target,
         metadata: crate::spec::TargetMetadata {
             description: Some("macOS with late-gen Intel (at least Haswell)".into()),
             tier: Some(3),
@@ -42,7 +38,7 @@ pub(crate) fn target() -> Target {
         pointer_width: 64,
         data_layout:
             "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128".into(),
-        arch: arch.target_arch(),
-        options: TargetOptions { mcount: "\u{1}mcount".into(), ..base },
+        arch,
+        options: TargetOptions { mcount: "\u{1}mcount".into(), ..opts },
     }
 }
diff --git a/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs b/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs
index bff2a184b19..65d21518491 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/infer/mod.rs
@@ -348,8 +348,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
                     }
                 }
                 if let Some(ty::error::ExpectedFound { found, .. }) = exp_found
-                    && ty.is_box()
-                    && ty.boxed_ty() == found
+                    && ty.boxed_ty() == Some(found)
                     && let Ok(snippet) = self.tcx.sess.source_map().span_to_snippet(span)
                 {
                     err.span_suggestion(
diff --git a/compiler/rustc_trait_selection/src/error_reporting/infer/need_type_info.rs b/compiler/rustc_trait_selection/src/error_reporting/infer/need_type_info.rs
index a93b633ce1a..cf9891f15ad 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/infer/need_type_info.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/infer/need_type_info.rs
@@ -1296,6 +1296,9 @@ impl<'a, 'tcx> Visitor<'tcx> for FindInferSourceVisitor<'a, 'tcx> {
             && let Some(def_id) = self.typeck_results.type_dependent_def_id(expr.hir_id)
             && self.tecx.tcx.trait_of_item(def_id).is_some()
             && !has_impl_trait(def_id)
+            // FIXME(fn_delegation): In delegation item argument spans are equal to last path
+            // segment. This leads to ICE's when emitting `multipart_suggestion`.
+            && tcx.hir().opt_delegation_sig_id(expr.hir_id.owner.def_id).is_none()
         {
             let successor =
                 method_args.get(0).map_or_else(|| (")", span.hi()), |arg| (", ", arg.span.lo()));
diff --git a/compiler/rustc_trait_selection/src/error_reporting/infer/note_and_explain.rs b/compiler/rustc_trait_selection/src/error_reporting/infer/note_and_explain.rs
index db71331d07f..7bfc6471dc8 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/infer/note_and_explain.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/infer/note_and_explain.rs
@@ -4,7 +4,7 @@ use rustc_hir as hir;
 use rustc_hir::def::DefKind;
 use rustc_middle::traits::{ObligationCause, ObligationCauseCode};
 use rustc_middle::ty::error::{ExpectedFound, TypeError};
-use rustc_middle::ty::fast_reject::{DeepRejectCtxt, TreatParams};
+use rustc_middle::ty::fast_reject::DeepRejectCtxt;
 use rustc_middle::ty::print::{FmtPrinter, Printer};
 use rustc_middle::ty::{self, suggest_constraining_type_param, Ty};
 use rustc_span::def_id::DefId;
@@ -317,7 +317,7 @@ impl<T> Trait<T> for X {
                     {
                         let mut has_matching_impl = false;
                         tcx.for_each_relevant_impl(def_id, values.found, |did| {
-                            if DeepRejectCtxt::new(tcx, TreatParams::ForLookup)
+                            if DeepRejectCtxt::relate_rigid_infer(tcx)
                                 .types_may_unify(values.found, tcx.type_of(did).skip_binder())
                             {
                                 has_matching_impl = true;
@@ -338,7 +338,7 @@ impl<T> Trait<T> for X {
                     {
                         let mut has_matching_impl = false;
                         tcx.for_each_relevant_impl(def_id, values.expected, |did| {
-                            if DeepRejectCtxt::new(tcx, TreatParams::ForLookup)
+                            if DeepRejectCtxt::relate_rigid_infer(tcx)
                                 .types_may_unify(values.expected, tcx.type_of(did).skip_binder())
                             {
                                 has_matching_impl = true;
@@ -358,7 +358,7 @@ impl<T> Trait<T> for X {
                     {
                         let mut has_matching_impl = false;
                         tcx.for_each_relevant_impl(def_id, values.found, |did| {
-                            if DeepRejectCtxt::new(tcx, TreatParams::ForLookup)
+                            if DeepRejectCtxt::relate_rigid_infer(tcx)
                                 .types_may_unify(values.found, tcx.type_of(did).skip_binder())
                             {
                                 has_matching_impl = true;
diff --git a/compiler/rustc_trait_selection/src/error_reporting/infer/region.rs b/compiler/rustc_trait_selection/src/error_reporting/infer/region.rs
index e4a4ec125a5..0372105a02b 100644
--- a/compiler/rustc_trait_selection/src/error_reporting/infer/region.rs
+++ b/compiler/rustc_trait_selection/src/error_reporting/infer/region.rs
@@ -625,11 +625,19 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
                 if let ObligationCauseCode::WhereClause(_, span)
                 | ObligationCauseCode::WhereClauseInExpr(_, span, ..) =
                     &trace.cause.code().peel_derives()
-                    && !span.is_dummy()
                 {
                     let span = *span;
-                    self.report_concrete_failure(generic_param_scope, placeholder_origin, sub, sup)
-                        .with_span_note(span, "the lifetime requirement is introduced here")
+                    let mut err = self.report_concrete_failure(
+                        generic_param_scope,
+                        placeholder_origin,
+                        sub,
+                        sup,
+                    );
+                    if !span.is_dummy() {
+                        err =
+                            err.with_span_note(span, "the lifetime requirement is introduced here");
+                    }
+                    err
                 } else {
                     unreachable!(
                         "control flow ensures we have a `BindingObligation` or `WhereClauseInExpr` here..."
@@ -1018,7 +1026,7 @@ impl<'a, 'tcx> TypeErrCtxt<'a, 'tcx> {
         let var_description = match var_origin {
             infer::MiscVariable(_) => String::new(),
             infer::PatternRegion(_) => " for pattern".to_string(),
-            infer::AddrOfRegion(_) => " for borrow expression".to_string(),
+            infer::BorrowRegion(_) => " for borrow expression".to_string(),
             infer::Autoref(_) => " for autoref".to_string(),
             infer::Coercion(_) => " for automatic coercion".to_string(),
             infer::BoundRegion(_, br, infer::FnCall) => {
diff --git a/compiler/rustc_trait_selection/src/traits/coherence.rs b/compiler/rustc_trait_selection/src/traits/coherence.rs
index 8b55f84bccc..bafe1ffae44 100644
--- a/compiler/rustc_trait_selection/src/traits/coherence.rs
+++ b/compiler/rustc_trait_selection/src/traits/coherence.rs
@@ -15,7 +15,7 @@ use rustc_middle::bug;
 use rustc_middle::traits::query::NoSolution;
 use rustc_middle::traits::solve::{CandidateSource, Certainty, Goal};
 use rustc_middle::traits::specialization_graph::OverlapMode;
-use rustc_middle::ty::fast_reject::{DeepRejectCtxt, TreatParams};
+use rustc_middle::ty::fast_reject::DeepRejectCtxt;
 use rustc_middle::ty::visit::{TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor};
 use rustc_middle::ty::{self, Ty, TyCtxt};
 pub use rustc_next_trait_solver::coherence::*;
@@ -96,7 +96,7 @@ pub fn overlapping_impls(
     // Before doing expensive operations like entering an inference context, do
     // a quick check via fast_reject to tell if the impl headers could possibly
     // unify.
-    let drcx = DeepRejectCtxt::new(tcx, TreatParams::AsCandidateKey);
+    let drcx = DeepRejectCtxt::relate_infer_infer(tcx);
     let impl1_ref = tcx.impl_trait_ref(impl1_def_id);
     let impl2_ref = tcx.impl_trait_ref(impl2_def_id);
     let may_overlap = match (impl1_ref, impl2_ref) {
diff --git a/compiler/rustc_trait_selection/src/traits/engine.rs b/compiler/rustc_trait_selection/src/traits/engine.rs
index de1d4ef15ac..c8811bc37b3 100644
--- a/compiler/rustc_trait_selection/src/traits/engine.rs
+++ b/compiler/rustc_trait_selection/src/traits/engine.rs
@@ -35,10 +35,8 @@ where
         if infcx.next_trait_solver() {
             Box::new(NextFulfillmentCtxt::new(infcx))
         } else {
-            let new_solver_globally =
-                infcx.tcx.sess.opts.unstable_opts.next_solver.map_or(false, |c| c.globally);
             assert!(
-                !new_solver_globally,
+                !infcx.tcx.next_trait_solver_globally(),
                 "using old solver even though new solver is enabled globally"
             );
             Box::new(FulfillmentContext::new(infcx))
diff --git a/compiler/rustc_trait_selection/src/traits/project.rs b/compiler/rustc_trait_selection/src/traits/project.rs
index 4702fd866c1..630acc91fbe 100644
--- a/compiler/rustc_trait_selection/src/traits/project.rs
+++ b/compiler/rustc_trait_selection/src/traits/project.rs
@@ -13,6 +13,7 @@ use rustc_infer::traits::ObligationCauseCode;
 use rustc_middle::traits::select::OverflowError;
 pub use rustc_middle::traits::Reveal;
 use rustc_middle::traits::{BuiltinImplSource, ImplSource, ImplSourceUserDefinedData};
+use rustc_middle::ty::fast_reject::DeepRejectCtxt;
 use rustc_middle::ty::fold::TypeFoldable;
 use rustc_middle::ty::visit::{MaxUniverse, TypeVisitable, TypeVisitableExt};
 use rustc_middle::ty::{self, Term, Ty, TyCtxt, Upcast};
@@ -886,6 +887,7 @@ fn assemble_candidates_from_predicates<'cx, 'tcx>(
     potentially_unnormalized_candidates: bool,
 ) {
     let infcx = selcx.infcx;
+    let drcx = DeepRejectCtxt::relate_rigid_rigid(selcx.tcx());
     for predicate in env_predicates {
         let bound_predicate = predicate.kind();
         if let ty::ClauseKind::Projection(data) = predicate.kind().skip_binder() {
@@ -894,6 +896,12 @@ fn assemble_candidates_from_predicates<'cx, 'tcx>(
                 continue;
             }
 
+            if !drcx
+                .args_may_unify(obligation.predicate.args, data.skip_binder().projection_term.args)
+            {
+                continue;
+            }
+
             let is_match = infcx.probe(|_| {
                 selcx.match_projection_projections(
                     obligation,
diff --git a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
index 96faa5236b1..77efc2fc2db 100644
--- a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
@@ -13,7 +13,7 @@ use hir::LangItem;
 use rustc_data_structures::fx::{FxHashSet, FxIndexSet};
 use rustc_hir as hir;
 use rustc_infer::traits::{Obligation, ObligationCause, PolyTraitObligation, SelectionError};
-use rustc_middle::ty::fast_reject::{DeepRejectCtxt, TreatParams};
+use rustc_middle::ty::fast_reject::DeepRejectCtxt;
 use rustc_middle::ty::{self, ToPolyTraitRef, Ty, TypeVisitableExt};
 use rustc_middle::{bug, span_bug};
 use tracing::{debug, instrument, trace};
@@ -248,11 +248,17 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             .filter(|p| p.def_id() == stack.obligation.predicate.def_id())
             .filter(|p| p.polarity() == stack.obligation.predicate.polarity());
 
+        let drcx = DeepRejectCtxt::relate_rigid_rigid(self.tcx());
+        let obligation_args = stack.obligation.predicate.skip_binder().trait_ref.args;
         // Keep only those bounds which may apply, and propagate overflow if it occurs.
         for bound in bounds {
+            let bound_trait_ref = bound.map_bound(|t| t.trait_ref);
+            if !drcx.args_may_unify(obligation_args, bound_trait_ref.skip_binder().args) {
+                continue;
+            }
             // FIXME(oli-obk): it is suspicious that we are dropping the constness and
             // polarity here.
-            let wc = self.where_clause_may_apply(stack, bound.map_bound(|t| t.trait_ref))?;
+            let wc = self.where_clause_may_apply(stack, bound_trait_ref)?;
             if wc.may_apply() {
                 candidates.vec.push(ParamCandidate(bound));
             }
@@ -581,7 +587,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             return;
         }
 
-        let drcx = DeepRejectCtxt::new(self.tcx(), TreatParams::ForLookup);
+        let drcx = DeepRejectCtxt::relate_rigid_infer(self.tcx());
         let obligation_args = obligation.predicate.skip_binder().trait_ref.args;
         self.tcx().for_each_relevant_impl(
             obligation.predicate.def_id(),
diff --git a/compiler/rustc_trait_selection/src/traits/specialize/specialization_graph.rs b/compiler/rustc_trait_selection/src/traits/specialize/specialization_graph.rs
index 0fdaf40b136..13620f4b8d9 100644
--- a/compiler/rustc_trait_selection/src/traits/specialize/specialization_graph.rs
+++ b/compiler/rustc_trait_selection/src/traits/specialize/specialization_graph.rs
@@ -41,7 +41,7 @@ impl<'tcx> Children {
     fn insert_blindly(&mut self, tcx: TyCtxt<'tcx>, impl_def_id: DefId) {
         let trait_ref = tcx.impl_trait_ref(impl_def_id).unwrap().skip_binder();
         if let Some(st) =
-            fast_reject::simplify_type(tcx, trait_ref.self_ty(), TreatParams::AsCandidateKey)
+            fast_reject::simplify_type(tcx, trait_ref.self_ty(), TreatParams::InstantiateWithInfer)
         {
             debug!("insert_blindly: impl_def_id={:?} st={:?}", impl_def_id, st);
             self.non_blanket_impls.entry(st).or_default().push(impl_def_id)
@@ -58,7 +58,7 @@ impl<'tcx> Children {
         let trait_ref = tcx.impl_trait_ref(impl_def_id).unwrap().skip_binder();
         let vec: &mut Vec<DefId>;
         if let Some(st) =
-            fast_reject::simplify_type(tcx, trait_ref.self_ty(), TreatParams::AsCandidateKey)
+            fast_reject::simplify_type(tcx, trait_ref.self_ty(), TreatParams::InstantiateWithInfer)
         {
             debug!("remove_existing: impl_def_id={:?} st={:?}", impl_def_id, st);
             vec = self.non_blanket_impls.get_mut(&st).unwrap();
@@ -279,7 +279,7 @@ impl<'tcx> Graph {
         let mut parent = trait_def_id;
         let mut last_lint = None;
         let simplified =
-            fast_reject::simplify_type(tcx, trait_ref.self_ty(), TreatParams::AsCandidateKey);
+            fast_reject::simplify_type(tcx, trait_ref.self_ty(), TreatParams::InstantiateWithInfer);
 
         // Descend the specialization tree, where `parent` is the current parent node.
         loop {
diff --git a/compiler/rustc_transmute/src/layout/nfa.rs b/compiler/rustc_transmute/src/layout/nfa.rs
index 5db5a8f222d..3b49c59fa68 100644
--- a/compiler/rustc_transmute/src/layout/nfa.rs
+++ b/compiler/rustc_transmute/src/layout/nfa.rs
@@ -87,8 +87,6 @@ where
     pub(crate) fn from_tree(tree: Tree<!, R>) -> Result<Self, Uninhabited> {
         Ok(match tree {
             Tree::Byte(b) => Self::from_byte(b),
-            #[cfg(bootstrap)]
-            Tree::Def(..) => unreachable!(),
             Tree::Ref(r) => Self::from_ref(r),
             Tree::Alt(alts) => {
                 let mut alts = alts.into_iter().map(Self::from_tree);
diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs
index 234e1a6d55e..c23a7fec5a5 100644
--- a/compiler/rustc_ty_utils/src/layout.rs
+++ b/compiler/rustc_ty_utils/src/layout.rs
@@ -1001,7 +1001,13 @@ fn coroutine_layout<'tcx>(
         },
         fields: outer_fields,
         abi,
-        largest_niche: prefix.largest_niche,
+        // Suppress niches inside coroutines. If the niche is inside a field that is aliased (due to
+        // self-referentiality), getting the discriminant can cause aliasing violations.
+        // `UnsafeCell` blocks niches for the same reason, but we don't yet have `UnsafePinned` that
+        // would do the same for us here.
+        // See <https://github.com/rust-lang/rust/issues/63818>, <https://github.com/rust-lang/miri/issues/3780>.
+        // FIXME: Remove when <https://github.com/rust-lang/rust/issues/125735> is implemented and aliased coroutine fields are wrapped in `UnsafePinned`.
+        largest_niche: None,
         size,
         align,
         max_repr_align: None,
diff --git a/compiler/rustc_type_ir/src/fast_reject.rs b/compiler/rustc_type_ir/src/fast_reject.rs
index fab4a099117..2c8e47bcbca 100644
--- a/compiler/rustc_type_ir/src/fast_reject.rs
+++ b/compiler/rustc_type_ir/src/fast_reject.rs
@@ -74,13 +74,13 @@ impl<HCX: Clone, DefId: HashStable<HCX>> ToStableHashKey<HCX> for SimplifiedType
 pub enum TreatParams {
     /// Treat parameters as infer vars. This is the correct mode for caching
     /// an impl's type for lookup.
-    AsCandidateKey,
+    InstantiateWithInfer,
     /// Treat parameters as placeholders in the given environment. This is the
     /// correct mode for *lookup*, as during candidate selection.
     ///
     /// This also treats projections with inference variables as infer vars
     /// since they could be further normalized.
-    ForLookup,
+    AsRigid,
 }
 
 /// Tries to simplify a type by only returning the outermost injective¹ layer, if one exists.
@@ -140,18 +140,16 @@ pub fn simplify_type<I: Interner>(
         }
         ty::Placeholder(..) => Some(SimplifiedType::Placeholder),
         ty::Param(_) => match treat_params {
-            TreatParams::ForLookup => Some(SimplifiedType::Placeholder),
-            TreatParams::AsCandidateKey => None,
+            TreatParams::AsRigid => Some(SimplifiedType::Placeholder),
+            TreatParams::InstantiateWithInfer => None,
         },
         ty::Alias(..) => match treat_params {
             // When treating `ty::Param` as a placeholder, projections also
             // don't unify with anything else as long as they are fully normalized.
             // FIXME(-Znext-solver): Can remove this `if` and always simplify to `Placeholder`
             // when the new solver is enabled by default.
-            TreatParams::ForLookup if !ty.has_non_region_infer() => {
-                Some(SimplifiedType::Placeholder)
-            }
-            TreatParams::ForLookup | TreatParams::AsCandidateKey => None,
+            TreatParams::AsRigid if !ty.has_non_region_infer() => Some(SimplifiedType::Placeholder),
+            TreatParams::AsRigid | TreatParams::InstantiateWithInfer => None,
         },
         ty::Foreign(def_id) => Some(SimplifiedType::Foreign(def_id)),
         ty::Error(_) => Some(SimplifiedType::Error),
@@ -173,29 +171,49 @@ impl<DefId> SimplifiedType<DefId> {
     }
 }
 
-/// Given generic arguments from an obligation and an impl,
-/// could these two be unified after replacing parameters in the
-/// the impl with inference variables.
+/// Given generic arguments, could they be unified after
+/// replacing parameters with inference variables or placeholders.
+/// This behavior is toggled using the const generics.
 ///
-/// For obligations, parameters won't be replaced by inference
-/// variables and only unify with themselves. We treat them
-/// the same way we treat placeholders.
+/// We use this to quickly reject impl/wc candidates without needing
+/// to instantiate generic arguments/having to enter a probe.
 ///
 /// We also use this function during coherence. For coherence the
 /// impls only have to overlap for some value, so we treat parameters
-/// on both sides like inference variables. This behavior is toggled
-/// using the `treat_obligation_params` field.
+/// on both sides like inference variables.
 #[derive(Debug, Clone, Copy)]
-pub struct DeepRejectCtxt<I: Interner> {
-    treat_obligation_params: TreatParams,
+pub struct DeepRejectCtxt<
+    I: Interner,
+    const INSTANTIATE_LHS_WITH_INFER: bool,
+    const INSTANTIATE_RHS_WITH_INFER: bool,
+> {
     _interner: PhantomData<I>,
 }
 
-impl<I: Interner> DeepRejectCtxt<I> {
-    pub fn new(_interner: I, treat_obligation_params: TreatParams) -> Self {
-        DeepRejectCtxt { treat_obligation_params, _interner: PhantomData }
+impl<I: Interner> DeepRejectCtxt<I, false, false> {
+    /// Treat parameters in both the lhs and the rhs as rigid.
+    pub fn relate_rigid_rigid(_interner: I) -> DeepRejectCtxt<I, false, false> {
+        DeepRejectCtxt { _interner: PhantomData }
+    }
+}
+
+impl<I: Interner> DeepRejectCtxt<I, true, true> {
+    /// Treat parameters in both the lhs and the rhs as infer vars.
+    pub fn relate_infer_infer(_interner: I) -> DeepRejectCtxt<I, true, true> {
+        DeepRejectCtxt { _interner: PhantomData }
     }
+}
 
+impl<I: Interner> DeepRejectCtxt<I, false, true> {
+    /// Treat parameters in the lhs as rigid, and in rhs as infer vars.
+    pub fn relate_rigid_infer(_interner: I) -> DeepRejectCtxt<I, false, true> {
+        DeepRejectCtxt { _interner: PhantomData }
+    }
+}
+
+impl<I: Interner, const INSTANTIATE_LHS_WITH_INFER: bool, const INSTANTIATE_RHS_WITH_INFER: bool>
+    DeepRejectCtxt<I, INSTANTIATE_LHS_WITH_INFER, INSTANTIATE_RHS_WITH_INFER>
+{
     pub fn args_may_unify(
         self,
         obligation_args: I::GenericArgs,
@@ -216,11 +234,18 @@ impl<I: Interner> DeepRejectCtxt<I> {
         })
     }
 
-    pub fn types_may_unify(self, obligation_ty: I::Ty, impl_ty: I::Ty) -> bool {
-        match impl_ty.kind() {
-            // Start by checking whether the type in the impl may unify with
+    pub fn types_may_unify(self, lhs: I::Ty, rhs: I::Ty) -> bool {
+        match rhs.kind() {
+            // Start by checking whether the `rhs` type may unify with
             // pretty much everything. Just return `true` in that case.
-            ty::Param(_) | ty::Error(_) | ty::Alias(..) => return true,
+            ty::Param(_) => {
+                if INSTANTIATE_RHS_WITH_INFER {
+                    return true;
+                }
+            }
+            ty::Error(_) | ty::Alias(..) | ty::Bound(..) => return true,
+            ty::Infer(var) => return self.var_and_ty_may_unify(var, lhs),
+
             // These types only unify with inference variables or their own
             // variant.
             ty::Bool
@@ -238,159 +263,217 @@ impl<I: Interner> DeepRejectCtxt<I> {
             | ty::Ref(..)
             | ty::Never
             | ty::Tuple(..)
+            | ty::FnDef(..)
             | ty::FnPtr(..)
-            | ty::Foreign(..) => debug_assert!(impl_ty.is_known_rigid()),
-            ty::FnDef(..)
             | ty::Closure(..)
             | ty::CoroutineClosure(..)
             | ty::Coroutine(..)
             | ty::CoroutineWitness(..)
-            | ty::Placeholder(..)
-            | ty::Bound(..)
-            | ty::Infer(_) => panic!("unexpected impl_ty: {impl_ty:?}"),
-        }
+            | ty::Foreign(_)
+            | ty::Placeholder(_) => {}
+        };
 
-        let k = impl_ty.kind();
-        match obligation_ty.kind() {
-            // Purely rigid types, use structural equivalence.
-            ty::Bool
-            | ty::Char
-            | ty::Int(_)
-            | ty::Uint(_)
-            | ty::Float(_)
-            | ty::Str
-            | ty::Never
-            | ty::Foreign(_) => obligation_ty == impl_ty,
-            ty::Ref(_, obl_ty, obl_mutbl) => match k {
-                ty::Ref(_, impl_ty, impl_mutbl) => {
-                    obl_mutbl == impl_mutbl && self.types_may_unify(obl_ty, impl_ty)
+        // For purely rigid types, use structural equivalence.
+        match lhs.kind() {
+            ty::Ref(_, lhs_ty, lhs_mutbl) => match rhs.kind() {
+                ty::Ref(_, rhs_ty, rhs_mutbl) => {
+                    lhs_mutbl == rhs_mutbl && self.types_may_unify(lhs_ty, rhs_ty)
                 }
                 _ => false,
             },
-            ty::Adt(obl_def, obl_args) => match k {
-                ty::Adt(impl_def, impl_args) => {
-                    obl_def == impl_def && self.args_may_unify(obl_args, impl_args)
+
+            ty::Adt(lhs_def, lhs_args) => match rhs.kind() {
+                ty::Adt(rhs_def, rhs_args) => {
+                    lhs_def == rhs_def && self.args_may_unify(lhs_args, rhs_args)
                 }
                 _ => false,
             },
-            ty::Pat(obl_ty, _) => {
-                // FIXME(pattern_types): take pattern into account
-                matches!(k, ty::Pat(impl_ty, _) if self.types_may_unify(obl_ty, impl_ty))
+
+            // Depending on the value of const generics, we either treat generic parameters
+            // like placeholders or like inference variables.
+            ty::Param(lhs) => {
+                INSTANTIATE_LHS_WITH_INFER
+                    || match rhs.kind() {
+                        ty::Param(rhs) => lhs == rhs,
+                        _ => false,
+                    }
             }
-            ty::Slice(obl_ty) => {
-                matches!(k, ty::Slice(impl_ty) if self.types_may_unify(obl_ty, impl_ty))
+
+            // Placeholder types don't unify with anything on their own.
+            ty::Placeholder(lhs) => {
+                matches!(rhs.kind(), ty::Placeholder(rhs) if lhs == rhs)
             }
-            ty::Array(obl_ty, obl_len) => match k {
-                ty::Array(impl_ty, impl_len) => {
-                    self.types_may_unify(obl_ty, impl_ty)
-                        && self.consts_may_unify(obl_len, impl_len)
+
+            ty::Infer(var) => self.var_and_ty_may_unify(var, rhs),
+
+            // As we're walking the whole type, it may encounter projections
+            // inside of binders and what not, so we're just going to assume that
+            // projections can unify with other stuff.
+            //
+            // Looking forward to lazy normalization this is the safer strategy anyways.
+            ty::Alias(..) => true,
+
+            ty::Int(_)
+            | ty::Uint(_)
+            | ty::Float(_)
+            | ty::Str
+            | ty::Bool
+            | ty::Char
+            | ty::Never
+            | ty::Foreign(_) => lhs == rhs,
+
+            ty::Tuple(lhs) => match rhs.kind() {
+                ty::Tuple(rhs) => {
+                    lhs.len() == rhs.len()
+                        && iter::zip(lhs.iter(), rhs.iter())
+                            .all(|(lhs, rhs)| self.types_may_unify(lhs, rhs))
                 }
                 _ => false,
             },
-            ty::Tuple(obl) => match k {
-                ty::Tuple(imp) => {
-                    obl.len() == imp.len()
-                        && iter::zip(obl.iter(), imp.iter())
-                            .all(|(obl, imp)| self.types_may_unify(obl, imp))
+
+            ty::Array(lhs_ty, lhs_len) => match rhs.kind() {
+                ty::Array(rhs_ty, rhs_len) => {
+                    self.types_may_unify(lhs_ty, rhs_ty) && self.consts_may_unify(lhs_len, rhs_len)
                 }
                 _ => false,
             },
-            ty::RawPtr(obl_ty, obl_mutbl) => match k {
-                ty::RawPtr(imp_ty, imp_mutbl) => {
-                    obl_mutbl == imp_mutbl && self.types_may_unify(obl_ty, imp_ty)
+
+            ty::RawPtr(lhs_ty, lhs_mutbl) => match rhs.kind() {
+                ty::RawPtr(rhs_ty, rhs_mutbl) => {
+                    lhs_mutbl == rhs_mutbl && self.types_may_unify(lhs_ty, rhs_ty)
                 }
                 _ => false,
             },
-            ty::Dynamic(obl_preds, ..) => {
+
+            ty::Slice(lhs_ty) => {
+                matches!(rhs.kind(), ty::Slice(rhs_ty) if self.types_may_unify(lhs_ty, rhs_ty))
+            }
+
+            ty::Dynamic(lhs_preds, ..) => {
                 // Ideally we would walk the existential predicates here or at least
                 // compare their length. But considering that the relevant `Relate` impl
                 // actually sorts and deduplicates these, that doesn't work.
-                matches!(k, ty::Dynamic(impl_preds, ..) if
-                    obl_preds.principal_def_id() == impl_preds.principal_def_id()
+                matches!(rhs.kind(), ty::Dynamic(rhs_preds, ..) if
+                    lhs_preds.principal_def_id() == rhs_preds.principal_def_id()
                 )
             }
-            ty::FnPtr(obl_sig_tys, obl_hdr) => match k {
-                ty::FnPtr(impl_sig_tys, impl_hdr) => {
-                    let obl_sig_tys = obl_sig_tys.skip_binder().inputs_and_output;
-                    let impl_sig_tys = impl_sig_tys.skip_binder().inputs_and_output;
-
-                    obl_hdr == impl_hdr
-                        && obl_sig_tys.len() == impl_sig_tys.len()
-                        && iter::zip(obl_sig_tys.iter(), impl_sig_tys.iter())
-                            .all(|(obl, imp)| self.types_may_unify(obl, imp))
+
+            ty::FnPtr(lhs_sig_tys, lhs_hdr) => match rhs.kind() {
+                ty::FnPtr(rhs_sig_tys, rhs_hdr) => {
+                    let lhs_sig_tys = lhs_sig_tys.skip_binder().inputs_and_output;
+                    let rhs_sig_tys = rhs_sig_tys.skip_binder().inputs_and_output;
+
+                    lhs_hdr == rhs_hdr
+                        && lhs_sig_tys.len() == rhs_sig_tys.len()
+                        && iter::zip(lhs_sig_tys.iter(), rhs_sig_tys.iter())
+                            .all(|(lhs, rhs)| self.types_may_unify(lhs, rhs))
                 }
                 _ => false,
             },
 
-            // Impls cannot contain these types as these cannot be named directly.
-            ty::FnDef(..) | ty::Closure(..) | ty::CoroutineClosure(..) | ty::Coroutine(..) => false,
-
-            // Placeholder types don't unify with anything on their own
-            ty::Placeholder(..) | ty::Bound(..) => false,
+            ty::Bound(..) => true,
 
-            // Depending on the value of `treat_obligation_params`, we either
-            // treat generic parameters like placeholders or like inference variables.
-            ty::Param(_) => match self.treat_obligation_params {
-                TreatParams::ForLookup => false,
-                TreatParams::AsCandidateKey => true,
+            ty::FnDef(lhs_def_id, lhs_args) => match rhs.kind() {
+                ty::FnDef(rhs_def_id, rhs_args) => {
+                    lhs_def_id == rhs_def_id && self.args_may_unify(lhs_args, rhs_args)
+                }
+                _ => false,
             },
 
-            ty::Infer(ty::IntVar(_)) => impl_ty.is_integral(),
-
-            ty::Infer(ty::FloatVar(_)) => impl_ty.is_floating_point(),
+            ty::Closure(lhs_def_id, lhs_args) => match rhs.kind() {
+                ty::Closure(rhs_def_id, rhs_args) => {
+                    lhs_def_id == rhs_def_id && self.args_may_unify(lhs_args, rhs_args)
+                }
+                _ => false,
+            },
 
-            ty::Infer(_) => true,
+            ty::CoroutineClosure(lhs_def_id, lhs_args) => match rhs.kind() {
+                ty::CoroutineClosure(rhs_def_id, rhs_args) => {
+                    lhs_def_id == rhs_def_id && self.args_may_unify(lhs_args, rhs_args)
+                }
+                _ => false,
+            },
 
-            // As we're walking the whole type, it may encounter projections
-            // inside of binders and what not, so we're just going to assume that
-            // projections can unify with other stuff.
-            //
-            // Looking forward to lazy normalization this is the safer strategy anyways.
-            ty::Alias(..) => true,
+            ty::Coroutine(lhs_def_id, lhs_args) => match rhs.kind() {
+                ty::Coroutine(rhs_def_id, rhs_args) => {
+                    lhs_def_id == rhs_def_id && self.args_may_unify(lhs_args, rhs_args)
+                }
+                _ => false,
+            },
 
-            ty::Error(_) => true,
+            ty::CoroutineWitness(lhs_def_id, lhs_args) => match rhs.kind() {
+                ty::CoroutineWitness(rhs_def_id, rhs_args) => {
+                    lhs_def_id == rhs_def_id && self.args_may_unify(lhs_args, rhs_args)
+                }
+                _ => false,
+            },
 
-            ty::CoroutineWitness(..) => {
-                panic!("unexpected obligation type: {:?}", obligation_ty)
+            ty::Pat(lhs_ty, _) => {
+                // FIXME(pattern_types): take pattern into account
+                matches!(rhs.kind(), ty::Pat(rhs_ty, _) if self.types_may_unify(lhs_ty, rhs_ty))
             }
+
+            ty::Error(..) => true,
         }
     }
 
-    pub fn consts_may_unify(self, obligation_ct: I::Const, impl_ct: I::Const) -> bool {
-        let impl_val = match impl_ct.kind() {
+    pub fn consts_may_unify(self, lhs: I::Const, rhs: I::Const) -> bool {
+        match rhs.kind() {
+            ty::ConstKind::Param(_) => {
+                if INSTANTIATE_RHS_WITH_INFER {
+                    return true;
+                }
+            }
+
             ty::ConstKind::Expr(_)
-            | ty::ConstKind::Param(_)
             | ty::ConstKind::Unevaluated(_)
-            | ty::ConstKind::Error(_) => {
+            | ty::ConstKind::Error(_)
+            | ty::ConstKind::Infer(_)
+            | ty::ConstKind::Bound(..) => {
                 return true;
             }
-            ty::ConstKind::Value(_, impl_val) => impl_val,
-            ty::ConstKind::Infer(_) | ty::ConstKind::Bound(..) | ty::ConstKind::Placeholder(_) => {
-                panic!("unexpected impl arg: {:?}", impl_ct)
-            }
+
+            ty::ConstKind::Value(..) | ty::ConstKind::Placeholder(_) => {}
         };
 
-        match obligation_ct.kind() {
-            ty::ConstKind::Param(_) => match self.treat_obligation_params {
-                TreatParams::ForLookup => false,
-                TreatParams::AsCandidateKey => true,
+        match lhs.kind() {
+            ty::ConstKind::Value(_, lhs_val) => match rhs.kind() {
+                ty::ConstKind::Value(_, rhs_val) => lhs_val == rhs_val,
+                _ => false,
             },
 
+            ty::ConstKind::Param(lhs) => {
+                INSTANTIATE_LHS_WITH_INFER
+                    || match rhs.kind() {
+                        ty::ConstKind::Param(rhs) => lhs == rhs,
+                        _ => false,
+                    }
+            }
+
             // Placeholder consts don't unify with anything on their own
-            ty::ConstKind::Placeholder(_) => false,
+            ty::ConstKind::Placeholder(lhs) => {
+                matches!(rhs.kind(), ty::ConstKind::Placeholder(rhs) if lhs == rhs)
+            }
 
             // As we don't necessarily eagerly evaluate constants,
             // they might unify with any value.
             ty::ConstKind::Expr(_) | ty::ConstKind::Unevaluated(_) | ty::ConstKind::Error(_) => {
                 true
             }
-            ty::ConstKind::Value(_, obl_val) => obl_val == impl_val,
 
-            ty::ConstKind::Infer(_) => true,
+            ty::ConstKind::Infer(_) | ty::ConstKind::Bound(..) => true,
+        }
+    }
 
-            ty::ConstKind::Bound(..) => {
-                panic!("unexpected obl const: {:?}", obligation_ct)
-            }
+    fn var_and_ty_may_unify(self, var: ty::InferTy, ty: I::Ty) -> bool {
+        if !ty.is_known_rigid() {
+            return true;
+        }
+
+        match var {
+            ty::IntVar(_) => ty.is_integral(),
+            ty::FloatVar(_) => ty.is_floating_point(),
+            _ => true,
         }
     }
 }
diff --git a/compiler/rustc_type_ir/src/fold.rs b/compiler/rustc_type_ir/src/fold.rs
index f49aa3431b5..64ce6fd69e6 100644
--- a/compiler/rustc_type_ir/src/fold.rs
+++ b/compiler/rustc_type_ir/src/fold.rs
@@ -91,8 +91,6 @@ pub trait TypeFoldable<I: Interner>: TypeVisitable<I> {
     fn fold_with<F: TypeFolder<I>>(self, folder: &mut F) -> Self {
         match self.try_fold_with(folder) {
             Ok(t) => t,
-            #[cfg(bootstrap)]
-            Err(e) => match e {},
         }
     }
 }
@@ -116,8 +114,6 @@ pub trait TypeSuperFoldable<I: Interner>: TypeFoldable<I> {
     fn super_fold_with<F: TypeFolder<I>>(self, folder: &mut F) -> Self {
         match self.try_super_fold_with(folder) {
             Ok(t) => t,
-            #[cfg(bootstrap)]
-            Err(e) => match e {},
         }
     }
 }