Merge pull request #2519 from rust-lang/rustc-pull

Rustc pull update

141 files changed, 2454 insertions, 2234 deletions

diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs
index 80b44e432ee..716bb716cdb 100644
--- a/compiler/rustc_abi/src/layout.rs
+++ b/compiler/rustc_abi/src/layout.rs
@@ -313,7 +313,6 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
         scalar_valid_range: (Bound<u128>, Bound<u128>),
         discr_range_of_repr: impl Fn(i128, i128) -> (Integer, bool),
         discriminants: impl Iterator<Item = (VariantIdx, i128)>,
-        dont_niche_optimize_enum: bool,
         always_sized: bool,
     ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {
         let (present_first, present_second) = {
@@ -352,13 +351,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
             // structs. (We have also handled univariant enums
             // that allow representation optimization.)
             assert!(is_enum);
-            self.layout_of_enum(
-                repr,
-                variants,
-                discr_range_of_repr,
-                discriminants,
-                dont_niche_optimize_enum,
-            )
+            self.layout_of_enum(repr, variants, discr_range_of_repr, discriminants)
         }
     }
 
@@ -599,7 +592,6 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
         variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,
         discr_range_of_repr: impl Fn(i128, i128) -> (Integer, bool),
         discriminants: impl Iterator<Item = (VariantIdx, i128)>,
-        dont_niche_optimize_enum: bool,
     ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {
         // Until we've decided whether to use the tagged or
         // niche filling LayoutData, we don't want to intern the
@@ -618,7 +610,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
         }
 
         let calculate_niche_filling_layout = || -> Option<TmpLayout<FieldIdx, VariantIdx>> {
-            if dont_niche_optimize_enum {
+            if repr.inhibit_enum_layout_opt() {
                 return None;
             }
 
diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs
index 5bd73502d98..8e346706877 100644
--- a/compiler/rustc_abi/src/lib.rs
+++ b/compiler/rustc_abi/src/lib.rs
@@ -1376,6 +1376,28 @@ impl WrappingRange {
         }
     }
 
+    /// Returns `true` if all the values in `other` are contained in this range,
+    /// when the values are considered as having width `size`.
+    #[inline(always)]
+    pub fn contains_range(&self, other: Self, size: Size) -> bool {
+        if self.is_full_for(size) {
+            true
+        } else {
+            let trunc = |x| size.truncate(x);
+
+            let delta = self.start;
+            let max = trunc(self.end.wrapping_sub(delta));
+
+            let other_start = trunc(other.start.wrapping_sub(delta));
+            let other_end = trunc(other.end.wrapping_sub(delta));
+
+            // Having shifted both input ranges by `delta`, now we only need to check
+            // whether `0..=max` contains `other_start..=other_end`, which can only
+            // happen if the other doesn't wrap since `self` isn't everything.
+            (other_start <= other_end) && (other_end <= max)
+        }
+    }
+
     /// Returns `self` with replaced `start`
     #[inline(always)]
     fn with_start(mut self, start: u128) -> Self {
diff --git a/compiler/rustc_abi/src/tests.rs b/compiler/rustc_abi/src/tests.rs
index d993012378c..d49c2d44af8 100644
--- a/compiler/rustc_abi/src/tests.rs
+++ b/compiler/rustc_abi/src/tests.rs
@@ -5,3 +5,66 @@ fn align_constants() {
     assert_eq!(Align::ONE, Align::from_bytes(1).unwrap());
     assert_eq!(Align::EIGHT, Align::from_bytes(8).unwrap());
 }
+
+#[test]
+fn wrapping_range_contains_range() {
+    let size16 = Size::from_bytes(16);
+
+    let a = WrappingRange { start: 10, end: 20 };
+    assert!(a.contains_range(a, size16));
+    assert!(a.contains_range(WrappingRange { start: 11, end: 19 }, size16));
+    assert!(a.contains_range(WrappingRange { start: 10, end: 10 }, size16));
+    assert!(a.contains_range(WrappingRange { start: 20, end: 20 }, size16));
+    assert!(!a.contains_range(WrappingRange { start: 10, end: 21 }, size16));
+    assert!(!a.contains_range(WrappingRange { start: 9, end: 20 }, size16));
+    assert!(!a.contains_range(WrappingRange { start: 4, end: 6 }, size16));
+    assert!(!a.contains_range(WrappingRange { start: 24, end: 26 }, size16));
+
+    assert!(!a.contains_range(WrappingRange { start: 16, end: 14 }, size16));
+
+    let b = WrappingRange { start: 20, end: 10 };
+    assert!(b.contains_range(b, size16));
+    assert!(b.contains_range(WrappingRange { start: 20, end: 20 }, size16));
+    assert!(b.contains_range(WrappingRange { start: 10, end: 10 }, size16));
+    assert!(b.contains_range(WrappingRange { start: 0, end: 10 }, size16));
+    assert!(b.contains_range(WrappingRange { start: 20, end: 30 }, size16));
+    assert!(b.contains_range(WrappingRange { start: 20, end: 9 }, size16));
+    assert!(b.contains_range(WrappingRange { start: 21, end: 10 }, size16));
+    assert!(b.contains_range(WrappingRange { start: 999, end: 9999 }, size16));
+    assert!(b.contains_range(WrappingRange { start: 999, end: 9 }, size16));
+    assert!(!b.contains_range(WrappingRange { start: 19, end: 19 }, size16));
+    assert!(!b.contains_range(WrappingRange { start: 11, end: 11 }, size16));
+    assert!(!b.contains_range(WrappingRange { start: 19, end: 11 }, size16));
+    assert!(!b.contains_range(WrappingRange { start: 11, end: 19 }, size16));
+
+    let f = WrappingRange { start: 0, end: u128::MAX };
+    assert!(f.contains_range(WrappingRange { start: 10, end: 20 }, size16));
+    assert!(f.contains_range(WrappingRange { start: 20, end: 10 }, size16));
+
+    let g = WrappingRange { start: 2, end: 1 };
+    assert!(g.contains_range(WrappingRange { start: 10, end: 20 }, size16));
+    assert!(g.contains_range(WrappingRange { start: 20, end: 10 }, size16));
+
+    let size1 = Size::from_bytes(1);
+    let u8r = WrappingRange { start: 0, end: 255 };
+    let i8r = WrappingRange { start: 128, end: 127 };
+    assert!(u8r.contains_range(i8r, size1));
+    assert!(i8r.contains_range(u8r, size1));
+    assert!(!u8r.contains_range(i8r, size16));
+    assert!(i8r.contains_range(u8r, size16));
+
+    let boolr = WrappingRange { start: 0, end: 1 };
+    assert!(u8r.contains_range(boolr, size1));
+    assert!(i8r.contains_range(boolr, size1));
+    assert!(!boolr.contains_range(u8r, size1));
+    assert!(!boolr.contains_range(i8r, size1));
+
+    let cmpr = WrappingRange { start: 255, end: 1 };
+    assert!(u8r.contains_range(cmpr, size1));
+    assert!(i8r.contains_range(cmpr, size1));
+    assert!(!cmpr.contains_range(u8r, size1));
+    assert!(!cmpr.contains_range(i8r, size1));
+
+    assert!(!boolr.contains_range(cmpr, size1));
+    assert!(cmpr.contains_range(boolr, size1));
+}
diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
index 97e07095875..984b280e81b 100644
--- a/compiler/rustc_ast/src/ast.rs
+++ b/compiler/rustc_ast/src/ast.rs
@@ -2849,7 +2849,7 @@ impl InlineAsmOperand {
     }
 }
 
-#[derive(Clone, Copy, Encodable, Decodable, Debug, HashStable_Generic, Walkable)]
+#[derive(Clone, Copy, Encodable, Decodable, Debug, HashStable_Generic, Walkable, PartialEq, Eq)]
 pub enum AsmMacro {
     /// The `asm!` macro
     Asm,
diff --git a/compiler/rustc_attr_parsing/src/attributes/stability.rs b/compiler/rustc_attr_parsing/src/attributes/stability.rs
index 59337749c87..c54fc6b41f8 100644
--- a/compiler/rustc_attr_parsing/src/attributes/stability.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/stability.rs
@@ -74,8 +74,15 @@ impl<S: Stage> AttributeParser<S> for StabilityParser {
             template!(NameValueStr: "deprecation message"),
             |this, cx, args| {
                 reject_outside_std!(cx);
-                this.allowed_through_unstable_modules =
-                    args.name_value().and_then(|i| i.value_as_str())
+                let Some(nv) = args.name_value() else {
+                    cx.expected_name_value(cx.attr_span, None);
+                    return;
+                };
+                let Some(value_str) = nv.value_as_str() else {
+                    cx.expected_string_literal(nv.value_span, Some(nv.value_as_lit()));
+                    return;
+                };
+                this.allowed_through_unstable_modules = Some(value_str);
             },
         ),
     ];
@@ -247,7 +254,12 @@ pub(crate) fn parse_stability<S: Stage>(
     let mut feature = None;
     let mut since = None;
 
-    for param in args.list()?.mixed() {
+    let ArgParser::List(list) = args else {
+        cx.expected_list(cx.attr_span);
+        return None;
+    };
+
+    for param in list.mixed() {
         let param_span = param.span();
         let Some(param) = param.meta_item() else {
             cx.emit_err(session_diagnostics::UnsupportedLiteral {
@@ -322,7 +334,13 @@ pub(crate) fn parse_unstability<S: Stage>(
     let mut is_soft = false;
     let mut implied_by = None;
     let mut old_name = None;
-    for param in args.list()?.mixed() {
+
+    let ArgParser::List(list) = args else {
+        cx.expected_list(cx.attr_span);
+        return None;
+    };
+
+    for param in list.mixed() {
         let Some(param) = param.meta_item() else {
             cx.emit_err(session_diagnostics::UnsupportedLiteral {
                 span: param.span(),
diff --git a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
index 040a0607db5..7031a9bb3d4 100644
--- a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
@@ -1290,6 +1290,58 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                 span,
                 format!("if `{ty}` implemented `Clone`, you could clone the value"),
             );
+        } else if let ty::Adt(_, _) = ty.kind()
+            && let Some(clone_trait) = self.infcx.tcx.lang_items().clone_trait()
+        {
+            // For cases like `Option<NonClone>`, where `Option<T>: Clone` if `T: Clone`, we point
+            // at the types that should be `Clone`.
+            let ocx = ObligationCtxt::new_with_diagnostics(self.infcx);
+            let cause = ObligationCause::misc(expr.span, self.mir_def_id());
+            ocx.register_bound(cause, self.infcx.param_env, ty, clone_trait);
+            let errors = ocx.select_all_or_error();
+            if errors.iter().all(|error| {
+                match error.obligation.predicate.as_clause().and_then(|c| c.as_trait_clause()) {
+                    Some(clause) => match clause.self_ty().skip_binder().kind() {
+                        ty::Adt(def, _) => def.did().is_local() && clause.def_id() == clone_trait,
+                        _ => false,
+                    },
+                    None => false,
+                }
+            }) {
+                let mut type_spans = vec![];
+                let mut types = FxIndexSet::default();
+                for clause in errors
+                    .iter()
+                    .filter_map(|e| e.obligation.predicate.as_clause())
+                    .filter_map(|c| c.as_trait_clause())
+                {
+                    let ty::Adt(def, _) = clause.self_ty().skip_binder().kind() else { continue };
+                    type_spans.push(self.infcx.tcx.def_span(def.did()));
+                    types.insert(
+                        self.infcx
+                            .tcx
+                            .short_string(clause.self_ty().skip_binder(), &mut err.long_ty_path()),
+                    );
+                }
+                let mut span: MultiSpan = type_spans.clone().into();
+                for sp in type_spans {
+                    span.push_span_label(sp, "consider implementing `Clone` for this type");
+                }
+                span.push_span_label(expr.span, "you could clone this value");
+                let types: Vec<_> = types.into_iter().collect();
+                let msg = match &types[..] {
+                    [only] => format!("`{only}`"),
+                    [head @ .., last] => format!(
+                        "{} and `{last}`",
+                        head.iter().map(|t| format!("`{t}`")).collect::<Vec<_>>().join(", ")
+                    ),
+                    [] => unreachable!(),
+                };
+                err.span_note(
+                    span,
+                    format!("if {msg} implemented `Clone`, you could clone the value"),
+                );
+            }
         }
     }
 
diff --git a/compiler/rustc_borrowck/src/diagnostics/move_errors.rs b/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
index 447bf7d091a..a5661e44af8 100644
--- a/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
@@ -115,10 +115,8 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
     fn append_to_grouped_errors(
         &self,
         grouped_errors: &mut Vec<GroupedMoveError<'tcx>>,
-        error: MoveError<'tcx>,
+        MoveError { place: original_path, location, kind }: MoveError<'tcx>,
     ) {
-        let MoveError { place: original_path, location, kind } = error;
-
         // Note: that the only time we assign a place isn't a temporary
         // to a user variable is when initializing it.
         // If that ever stops being the case, then the ever initialized
@@ -251,54 +249,47 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
     }
 
     fn report(&mut self, error: GroupedMoveError<'tcx>) {
-        let (mut err, err_span) = {
-            let (span, use_spans, original_path, kind) = match error {
-                GroupedMoveError::MovesFromPlace { span, original_path, ref kind, .. }
-                | GroupedMoveError::MovesFromValue { span, original_path, ref kind, .. } => {
-                    (span, None, original_path, kind)
-                }
-                GroupedMoveError::OtherIllegalMove { use_spans, original_path, ref kind } => {
-                    (use_spans.args_or_use(), Some(use_spans), original_path, kind)
-                }
-            };
-            debug!(
-                "report: original_path={:?} span={:?}, kind={:?} \
-                   original_path.is_upvar_field_projection={:?}",
-                original_path,
-                span,
-                kind,
-                self.is_upvar_field_projection(original_path.as_ref())
-            );
-            if self.has_ambiguous_copy(original_path.ty(self.body, self.infcx.tcx).ty) {
-                // If the type may implement Copy, skip the error.
-                // It's an error with the Copy implementation (e.g. duplicate Copy) rather than borrow check
-                self.dcx().span_delayed_bug(
+        let (span, use_spans, original_path, kind) = match error {
+            GroupedMoveError::MovesFromPlace { span, original_path, ref kind, .. }
+            | GroupedMoveError::MovesFromValue { span, original_path, ref kind, .. } => {
+                (span, None, original_path, kind)
+            }
+            GroupedMoveError::OtherIllegalMove { use_spans, original_path, ref kind } => {
+                (use_spans.args_or_use(), Some(use_spans), original_path, kind)
+            }
+        };
+        debug!(
+            "report: original_path={:?} span={:?}, kind={:?} \
+             original_path.is_upvar_field_projection={:?}",
+            original_path,
+            span,
+            kind,
+            self.is_upvar_field_projection(original_path.as_ref())
+        );
+        if self.has_ambiguous_copy(original_path.ty(self.body, self.infcx.tcx).ty) {
+            // If the type may implement Copy, skip the error.
+            // It's an error with the Copy implementation (e.g. duplicate Copy) rather than borrow check
+            self.dcx()
+                .span_delayed_bug(span, "Type may implement copy, but there is no other error.");
+            return;
+        }
+        let mut err = match kind {
+            &IllegalMoveOriginKind::BorrowedContent { target_place } => self
+                .report_cannot_move_from_borrowed_content(
+                    original_path,
+                    target_place,
                     span,
-                    "Type may implement copy, but there is no other error.",
-                );
-                return;
+                    use_spans,
+                ),
+            &IllegalMoveOriginKind::InteriorOfTypeWithDestructor { container_ty: ty } => {
+                self.cannot_move_out_of_interior_of_drop(span, ty)
+            }
+            &IllegalMoveOriginKind::InteriorOfSliceOrArray { ty, is_index } => {
+                self.cannot_move_out_of_interior_noncopy(span, ty, Some(is_index))
             }
-            (
-                match kind {
-                    &IllegalMoveOriginKind::BorrowedContent { target_place } => self
-                        .report_cannot_move_from_borrowed_content(
-                            original_path,
-                            target_place,
-                            span,
-                            use_spans,
-                        ),
-                    &IllegalMoveOriginKind::InteriorOfTypeWithDestructor { container_ty: ty } => {
-                        self.cannot_move_out_of_interior_of_drop(span, ty)
-                    }
-                    &IllegalMoveOriginKind::InteriorOfSliceOrArray { ty, is_index } => {
-                        self.cannot_move_out_of_interior_noncopy(span, ty, Some(is_index))
-                    }
-                },
-                span,
-            )
         };
 
-        self.add_move_hints(error, &mut err, err_span);
+        self.add_move_hints(error, &mut err, span);
         self.buffer_error(err);
     }
 
@@ -483,7 +474,8 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                 self.cannot_move_out_of_interior_noncopy(span, ty, None)
             }
             ty::Closure(def_id, closure_args)
-                if def_id.as_local() == Some(self.mir_def_id()) && upvar_field.is_some() =>
+                if def_id.as_local() == Some(self.mir_def_id())
+                    && let Some(upvar_field) = upvar_field =>
             {
                 let closure_kind_ty = closure_args.as_closure().kind_ty();
                 let closure_kind = match closure_kind_ty.to_opt_closure_kind() {
@@ -496,7 +488,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                 let capture_description =
                     format!("captured variable in an `{closure_kind}` closure");
 
-                let upvar = &self.upvars[upvar_field.unwrap().index()];
+                let upvar = &self.upvars[upvar_field.index()];
                 let upvar_hir_id = upvar.get_root_variable();
                 let upvar_name = upvar.to_string(tcx);
                 let upvar_span = tcx.hir_span(upvar_hir_id);
@@ -606,7 +598,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
             }
             // No binding. Nothing to suggest.
             GroupedMoveError::OtherIllegalMove { ref original_path, use_spans, .. } => {
-                let use_span = use_spans.var_or_use();
+                let mut use_span = use_spans.var_or_use();
                 let place_ty = original_path.ty(self.body, self.infcx.tcx).ty;
                 let place_desc = match self.describe_place(original_path.as_ref()) {
                     Some(desc) => format!("`{desc}`"),
@@ -623,6 +615,36 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                     );
                 }
 
+                if let Some(upvar_field) = self
+                    .prefixes(original_path.as_ref(), PrefixSet::All)
+                    .find_map(|p| self.is_upvar_field_projection(p))
+                {
+                    // Look for the introduction of the original binding being moved.
+                    let upvar = &self.upvars[upvar_field.index()];
+                    let upvar_hir_id = upvar.get_root_variable();
+                    use_span = match self.infcx.tcx.parent_hir_node(upvar_hir_id) {
+                        hir::Node::Param(param) => {
+                            // Instead of pointing at the path where we access the value within a
+                            // closure, we point at the type of the outer `fn` argument.
+                            param.ty_span
+                        }
+                        hir::Node::LetStmt(stmt) => match (stmt.ty, stmt.init) {
+                            // We point at the type of the outer let-binding.
+                            (Some(ty), _) => ty.span,
+                            // We point at the initializer of the outer let-binding, but only if it
+                            // isn't something that spans multiple lines, like a closure, as the
+                            // ASCII art gets messy.
+                            (None, Some(init))
+                                if !self.infcx.tcx.sess.source_map().is_multiline(init.span) =>
+                            {
+                                init.span
+                            }
+                            _ => use_span,
+                        },
+                        _ => use_span,
+                    };
+                }
+
                 err.subdiagnostic(crate::session_diagnostics::TypeNoCopy::Label {
                     is_partial_move: false,
                     ty: place_ty,
@@ -630,12 +652,22 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                     span: use_span,
                 });
 
+                let mut pointed_at_span = false;
                 use_spans.args_subdiag(err, |args_span| {
+                    if args_span == span || args_span == use_span {
+                        pointed_at_span = true;
+                    }
                     crate::session_diagnostics::CaptureArgLabel::MoveOutPlace {
-                        place: place_desc,
+                        place: place_desc.clone(),
                         args_span,
                     }
                 });
+                if !pointed_at_span && use_span != span {
+                    err.subdiagnostic(crate::session_diagnostics::CaptureArgLabel::MoveOutPlace {
+                        place: place_desc,
+                        args_span: span,
+                    });
+                }
 
                 self.add_note_for_packed_struct_derive(err, original_path.local);
             }
diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs
index 85adf0f3716..a04cfa27237 100644
--- a/compiler/rustc_codegen_cranelift/src/constant.rs
+++ b/compiler/rustc_codegen_cranelift/src/constant.rs
@@ -74,7 +74,7 @@ pub(crate) fn codegen_tls_ref<'tcx>(
 pub(crate) fn eval_mir_constant<'tcx>(
     fx: &FunctionCx<'_, '_, 'tcx>,
     constant: &ConstOperand<'tcx>,
-) -> (ConstValue<'tcx>, Ty<'tcx>) {
+) -> (ConstValue, Ty<'tcx>) {
     let cv = fx.monomorphize(constant.const_);
     // This cannot fail because we checked all required_consts in advance.
     let val = cv
@@ -93,7 +93,7 @@ pub(crate) fn codegen_constant_operand<'tcx>(
 
 pub(crate) fn codegen_const_value<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
-    const_val: ConstValue<'tcx>,
+    const_val: ConstValue,
     ty: Ty<'tcx>,
 ) -> CValue<'tcx> {
     let layout = fx.layout_of(ty);
@@ -210,8 +210,7 @@ pub(crate) fn codegen_const_value<'tcx>(
                 .offset_i64(fx, i64::try_from(offset.bytes()).unwrap()),
             layout,
         ),
-        ConstValue::Slice { data, meta } => {
-            let alloc_id = fx.tcx.reserve_and_set_memory_alloc(data);
+        ConstValue::Slice { alloc_id, meta } => {
             let ptr = pointer_for_allocation(fx, alloc_id).get_addr(fx);
             let len = fx.bcx.ins().iconst(fx.pointer_type, meta as i64);
             CValue::by_val_pair(ptr, len, layout)
diff --git a/compiler/rustc_codegen_gcc/messages.ftl b/compiler/rustc_codegen_gcc/messages.ftl
index 55a28bc9493..a70ac08f01a 100644
--- a/compiler/rustc_codegen_gcc/messages.ftl
+++ b/compiler/rustc_codegen_gcc/messages.ftl
@@ -3,12 +3,4 @@ codegen_gcc_unwinding_inline_asm =
 
 codegen_gcc_copy_bitcode = failed to copy bitcode to object file: {$err}
 
-codegen_gcc_dynamic_linking_with_lto =
-    cannot prefer dynamic linking when performing LTO
-    .note = only 'staticlib', 'bin', and 'cdylib' outputs are supported with LTO
-
-codegen_gcc_lto_disallowed = lto can only be run for executables, cdylibs and static library outputs
-
-codegen_gcc_lto_dylib = lto cannot be used for `dylib` crate type without `-Zdylib-lto`
-
 codegen_gcc_lto_bitcode_from_rlib = failed to get bitcode from object file for LTO ({$gcc_err})
diff --git a/compiler/rustc_codegen_gcc/src/back/lto.rs b/compiler/rustc_codegen_gcc/src/back/lto.rs
index e554dd2500b..d558dfbc1c4 100644
--- a/compiler/rustc_codegen_gcc/src/back/lto.rs
+++ b/compiler/rustc_codegen_gcc/src/back/lto.rs
@@ -25,35 +25,21 @@ use std::sync::Arc;
 use gccjit::{Context, OutputKind};
 use object::read::archive::ArchiveFile;
 use rustc_codegen_ssa::back::lto::{SerializedModule, ThinModule, ThinShared};
-use rustc_codegen_ssa::back::symbol_export;
 use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput};
 use rustc_codegen_ssa::traits::*;
 use rustc_codegen_ssa::{ModuleCodegen, ModuleKind, looks_like_rust_object_file};
 use rustc_data_structures::memmap::Mmap;
 use rustc_errors::{DiagCtxtHandle, FatalError};
-use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::bug;
 use rustc_middle::dep_graph::WorkProduct;
-use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel};
-use rustc_session::config::{CrateType, Lto};
+use rustc_session::config::Lto;
 use rustc_target::spec::RelocModel;
 use tempfile::{TempDir, tempdir};
 
 use crate::back::write::save_temp_bitcode;
-use crate::errors::{DynamicLinkingWithLTO, LtoBitcodeFromRlib, LtoDisallowed, LtoDylib};
+use crate::errors::LtoBitcodeFromRlib;
 use crate::{GccCodegenBackend, GccContext, SyncContext, to_gcc_opt_level};
 
-pub fn crate_type_allows_lto(crate_type: CrateType) -> bool {
-    match crate_type {
-        CrateType::Executable
-        | CrateType::Dylib
-        | CrateType::Staticlib
-        | CrateType::Cdylib
-        | CrateType::Sdylib => true,
-        CrateType::Rlib | CrateType::ProcMacro => false,
-    }
-}
-
 struct LtoData {
     // TODO(antoyo): use symbols_below_threshold.
     //symbols_below_threshold: Vec<String>,
@@ -63,18 +49,9 @@ struct LtoData {
 
 fn prepare_lto(
     cgcx: &CodegenContext<GccCodegenBackend>,
+    each_linked_rlib_for_lto: &[PathBuf],
     dcx: DiagCtxtHandle<'_>,
 ) -> Result<LtoData, FatalError> {
-    let export_threshold = match cgcx.lto {
-        // We're just doing LTO for our one crate
-        Lto::ThinLocal => SymbolExportLevel::Rust,
-
-        // We're doing LTO for the entire crate graph
-        Lto::Fat | Lto::Thin => symbol_export::crates_export_threshold(&cgcx.crate_types),
-
-        Lto::No => panic!("didn't request LTO but we're doing LTO"),
-    };
-
     let tmp_path = match tempdir() {
         Ok(tmp_path) => tmp_path,
         Err(error) => {
@@ -83,20 +60,6 @@ fn prepare_lto(
         }
     };
 
-    let symbol_filter = &|&(ref name, info): &(String, SymbolExportInfo)| {
-        if info.level.is_below_threshold(export_threshold) || info.used {
-            Some(name.clone())
-        } else {
-            None
-        }
-    };
-    let exported_symbols = cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO");
-    let mut symbols_below_threshold = {
-        let _timer = cgcx.prof.generic_activity("GCC_lto_generate_symbols_below_threshold");
-        exported_symbols[&LOCAL_CRATE].iter().filter_map(symbol_filter).collect::<Vec<String>>()
-    };
-    info!("{} symbols to preserve in this crate", symbols_below_threshold.len());
-
     // If we're performing LTO for the entire crate graph, then for each of our
     // upstream dependencies, find the corresponding rlib and load the bitcode
     // from the archive.
@@ -105,32 +68,7 @@ fn prepare_lto(
     // with either fat or thin LTO
     let mut upstream_modules = Vec::new();
     if cgcx.lto != Lto::ThinLocal {
-        // Make sure we actually can run LTO
-        for crate_type in cgcx.crate_types.iter() {
-            if !crate_type_allows_lto(*crate_type) {
-                dcx.emit_err(LtoDisallowed);
-                return Err(FatalError);
-            }
-            if *crate_type == CrateType::Dylib && !cgcx.opts.unstable_opts.dylib_lto {
-                dcx.emit_err(LtoDylib);
-                return Err(FatalError);
-            }
-        }
-
-        if cgcx.opts.cg.prefer_dynamic && !cgcx.opts.unstable_opts.dylib_lto {
-            dcx.emit_err(DynamicLinkingWithLTO);
-            return Err(FatalError);
-        }
-
-        for &(cnum, ref path) in cgcx.each_linked_rlib_for_lto.iter() {
-            let exported_symbols =
-                cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO");
-            {
-                let _timer = cgcx.prof.generic_activity("GCC_lto_generate_symbols_below_threshold");
-                symbols_below_threshold
-                    .extend(exported_symbols[&cnum].iter().filter_map(symbol_filter));
-            }
-
+        for path in each_linked_rlib_for_lto {
             let archive_data = unsafe {
                 Mmap::map(File::open(path).expect("couldn't open rlib")).expect("couldn't map rlib")
             };
@@ -174,19 +112,18 @@ fn save_as_file(obj: &[u8], path: &Path) -> Result<(), LtoBitcodeFromRlib> {
 /// for further optimization.
 pub(crate) fn run_fat(
     cgcx: &CodegenContext<GccCodegenBackend>,
+    each_linked_rlib_for_lto: &[PathBuf],
     modules: Vec<FatLtoInput<GccCodegenBackend>>,
-    cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
 ) -> Result<ModuleCodegen<GccContext>, FatalError> {
     let dcx = cgcx.create_dcx();
     let dcx = dcx.handle();
-    let lto_data = prepare_lto(cgcx, dcx)?;
+    let lto_data = prepare_lto(cgcx, each_linked_rlib_for_lto, dcx)?;
     /*let symbols_below_threshold =
     lto_data.symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();*/
     fat_lto(
         cgcx,
         dcx,
         modules,
-        cached_modules,
         lto_data.upstream_modules,
         lto_data.tmp_path,
         //&lto_data.symbols_below_threshold,
@@ -197,7 +134,6 @@ fn fat_lto(
     cgcx: &CodegenContext<GccCodegenBackend>,
     _dcx: DiagCtxtHandle<'_>,
     modules: Vec<FatLtoInput<GccCodegenBackend>>,
-    cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
     mut serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>,
     tmp_path: TempDir,
     //symbols_below_threshold: &[String],
@@ -211,21 +147,12 @@ fn fat_lto(
     //   modules that are serialized in-memory.
     // * `in_memory` contains modules which are already parsed and in-memory,
     //   such as from multi-CGU builds.
-    //
-    // All of `cached_modules` (cached from previous incremental builds) can
-    // immediately go onto the `serialized_modules` modules list and then we can
-    // split the `modules` array into these two lists.
     let mut in_memory = Vec::new();
-    serialized_modules.extend(cached_modules.into_iter().map(|(buffer, wp)| {
-        info!("pushing cached module {:?}", wp.cgu_name);
-        (buffer, CString::new(wp.cgu_name).unwrap())
-    }));
     for module in modules {
         match module {
             FatLtoInput::InMemory(m) => in_memory.push(m),
             FatLtoInput::Serialized { name, buffer } => {
                 info!("pushing serialized module {:?}", name);
-                let buffer = SerializedModule::Local(buffer);
                 serialized_modules.push((buffer, CString::new(name).unwrap()));
             }
         }
@@ -356,12 +283,13 @@ impl ModuleBufferMethods for ModuleBuffer {
 /// can simply be copied over from the incr. comp. cache.
 pub(crate) fn run_thin(
     cgcx: &CodegenContext<GccCodegenBackend>,
+    each_linked_rlib_for_lto: &[PathBuf],
     modules: Vec<(String, ThinBuffer)>,
     cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
 ) -> Result<(Vec<ThinModule<GccCodegenBackend>>, Vec<WorkProduct>), FatalError> {
     let dcx = cgcx.create_dcx();
     let dcx = dcx.handle();
-    let lto_data = prepare_lto(cgcx, dcx)?;
+    let lto_data = prepare_lto(cgcx, each_linked_rlib_for_lto, dcx)?;
     if cgcx.opts.cg.linker_plugin_lto.enabled() {
         unreachable!(
             "We should never reach this case if the LTO step \
diff --git a/compiler/rustc_codegen_gcc/src/errors.rs b/compiler/rustc_codegen_gcc/src/errors.rs
index b7e7343460f..0aa16bd88b4 100644
--- a/compiler/rustc_codegen_gcc/src/errors.rs
+++ b/compiler/rustc_codegen_gcc/src/errors.rs
@@ -15,19 +15,6 @@ pub(crate) struct CopyBitcode {
 }
 
 #[derive(Diagnostic)]
-#[diag(codegen_gcc_dynamic_linking_with_lto)]
-#[note]
-pub(crate) struct DynamicLinkingWithLTO;
-
-#[derive(Diagnostic)]
-#[diag(codegen_gcc_lto_disallowed)]
-pub(crate) struct LtoDisallowed;
-
-#[derive(Diagnostic)]
-#[diag(codegen_gcc_lto_dylib)]
-pub(crate) struct LtoDylib;
-
-#[derive(Diagnostic)]
 #[diag(codegen_gcc_lto_bitcode_from_rlib)]
 pub(crate) struct LtoBitcodeFromRlib {
     pub gcc_err: String,
diff --git a/compiler/rustc_codegen_gcc/src/lib.rs b/compiler/rustc_codegen_gcc/src/lib.rs
index af416929ea7..71765c51138 100644
--- a/compiler/rustc_codegen_gcc/src/lib.rs
+++ b/compiler/rustc_codegen_gcc/src/lib.rs
@@ -81,6 +81,7 @@ mod type_of;
 use std::any::Any;
 use std::fmt::Debug;
 use std::ops::Deref;
+use std::path::PathBuf;
 #[cfg(not(feature = "master"))]
 use std::sync::atomic::AtomicBool;
 #[cfg(not(feature = "master"))]
@@ -358,23 +359,28 @@ impl WriteBackendMethods for GccCodegenBackend {
 
     fn run_and_optimize_fat_lto(
         cgcx: &CodegenContext<Self>,
+        // FIXME(bjorn3): Limit LTO exports to these symbols
+        _exported_symbols_for_lto: &[String],
+        each_linked_rlib_for_lto: &[PathBuf],
         modules: Vec<FatLtoInput<Self>>,
-        cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>,
         diff_fncs: Vec<AutoDiffItem>,
     ) -> Result<ModuleCodegen<Self::Module>, FatalError> {
         if !diff_fncs.is_empty() {
             unimplemented!();
         }
 
-        back::lto::run_fat(cgcx, modules, cached_modules)
+        back::lto::run_fat(cgcx, each_linked_rlib_for_lto, modules)
     }
 
     fn run_thin_lto(
         cgcx: &CodegenContext<Self>,
+        // FIXME(bjorn3): Limit LTO exports to these symbols
+        _exported_symbols_for_lto: &[String],
+        each_linked_rlib_for_lto: &[PathBuf],
         modules: Vec<(String, Self::ThinBuffer)>,
         cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>,
     ) -> Result<(Vec<ThinModule<Self>>, Vec<WorkProduct>), FatalError> {
-        back::lto::run_thin(cgcx, modules, cached_modules)
+        back::lto::run_thin(cgcx, each_linked_rlib_for_lto, modules, cached_modules)
     }
 
     fn print_pass_timings(&self) {
diff --git a/compiler/rustc_codegen_llvm/messages.ftl b/compiler/rustc_codegen_llvm/messages.ftl
index f197ea74473..3d5f17a6034 100644
--- a/compiler/rustc_codegen_llvm/messages.ftl
+++ b/compiler/rustc_codegen_llvm/messages.ftl
@@ -2,10 +2,6 @@ codegen_llvm_autodiff_without_enable = using the autodiff feature requires -Z au
 
 codegen_llvm_copy_bitcode = failed to copy bitcode to object file: {$err}
 
-codegen_llvm_dynamic_linking_with_lto =
-    cannot prefer dynamic linking when performing LTO
-    .note = only 'staticlib', 'bin', and 'cdylib' outputs are supported with LTO
-
 
 codegen_llvm_fixed_x18_invalid_arch = the `-Zfixed-x18` flag is not supported on the `{$arch}` architecture
 
@@ -18,12 +14,6 @@ codegen_llvm_load_bitcode_with_llvm_err = failed to load bitcode of module "{$na
 
 codegen_llvm_lto_bitcode_from_rlib = failed to get bitcode from object file for LTO ({$llvm_err})
 
-codegen_llvm_lto_disallowed = lto can only be run for executables, cdylibs and static library outputs
-
-codegen_llvm_lto_dylib = lto cannot be used for `dylib` crate type without `-Zdylib-lto`
-
-codegen_llvm_lto_proc_macro = lto cannot be used for `proc-macro` crate type without `-Zdylib-lto`
-
 codegen_llvm_mismatch_data_layout =
     data-layout for target `{$rustc_target}`, `{$rustc_layout}`, differs from LLVM target's `{$llvm_target}` default layout, `{$llvm_layout}`
 
diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs
index 84302009da9..767835c34f0 100644
--- a/compiler/rustc_codegen_llvm/src/back/lto.rs
+++ b/compiler/rustc_codegen_llvm/src/back/lto.rs
@@ -1,33 +1,28 @@
 use std::collections::BTreeMap;
 use std::ffi::{CStr, CString};
 use std::fs::File;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::ptr::NonNull;
 use std::sync::Arc;
 use std::{io, iter, slice};
 
 use object::read::archive::ArchiveFile;
 use rustc_codegen_ssa::back::lto::{SerializedModule, ThinModule, ThinShared};
-use rustc_codegen_ssa::back::symbol_export;
 use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput};
 use rustc_codegen_ssa::traits::*;
 use rustc_codegen_ssa::{ModuleCodegen, ModuleKind, looks_like_rust_object_file};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::memmap::Mmap;
 use rustc_errors::{DiagCtxtHandle, FatalError};
-use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::bug;
 use rustc_middle::dep_graph::WorkProduct;
-use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel};
-use rustc_session::config::{self, CrateType, Lto};
+use rustc_session::config::{self, Lto};
 use tracing::{debug, info};
 
 use crate::back::write::{
     self, CodegenDiagnosticsStage, DiagnosticHandlers, bitcode_section_name, save_temp_bitcode,
 };
-use crate::errors::{
-    DynamicLinkingWithLTO, LlvmError, LtoBitcodeFromRlib, LtoDisallowed, LtoDylib, LtoProcMacro,
-};
+use crate::errors::{LlvmError, LtoBitcodeFromRlib};
 use crate::llvm::AttributePlace::Function;
 use crate::llvm::{self, build_string};
 use crate::{LlvmCodegenBackend, ModuleLlvm, SimpleCx, attributes};
@@ -36,45 +31,21 @@ use crate::{LlvmCodegenBackend, ModuleLlvm, SimpleCx, attributes};
 /// session to determine which CGUs we can reuse.
 const THIN_LTO_KEYS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-keys.bin";
 
-fn crate_type_allows_lto(crate_type: CrateType) -> bool {
-    match crate_type {
-        CrateType::Executable
-        | CrateType::Dylib
-        | CrateType::Staticlib
-        | CrateType::Cdylib
-        | CrateType::ProcMacro
-        | CrateType::Sdylib => true,
-        CrateType::Rlib => false,
-    }
-}
-
 fn prepare_lto(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
+    exported_symbols_for_lto: &[String],
+    each_linked_rlib_for_lto: &[PathBuf],
     dcx: DiagCtxtHandle<'_>,
 ) -> Result<(Vec<CString>, Vec<(SerializedModule<ModuleBuffer>, CString)>), FatalError> {
-    let export_threshold = match cgcx.lto {
-        // We're just doing LTO for our one crate
-        Lto::ThinLocal => SymbolExportLevel::Rust,
-
-        // We're doing LTO for the entire crate graph
-        Lto::Fat | Lto::Thin => symbol_export::crates_export_threshold(&cgcx.crate_types),
-
-        Lto::No => panic!("didn't request LTO but we're doing LTO"),
-    };
+    let mut symbols_below_threshold = exported_symbols_for_lto
+        .iter()
+        .map(|symbol| CString::new(symbol.to_owned()).unwrap())
+        .collect::<Vec<CString>>();
 
-    let symbol_filter = &|&(ref name, info): &(String, SymbolExportInfo)| {
-        if info.level.is_below_threshold(export_threshold) || info.used {
-            Some(CString::new(name.as_str()).unwrap())
-        } else {
-            None
-        }
-    };
-    let exported_symbols = cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO");
-    let mut symbols_below_threshold = {
-        let _timer = cgcx.prof.generic_activity("LLVM_lto_generate_symbols_below_threshold");
-        exported_symbols[&LOCAL_CRATE].iter().filter_map(symbol_filter).collect::<Vec<CString>>()
-    };
-    info!("{} symbols to preserve in this crate", symbols_below_threshold.len());
+    // __llvm_profile_counter_bias is pulled in at link time by an undefined reference to
+    // __llvm_profile_runtime, therefore we won't know until link time if this symbol
+    // should have default visibility.
+    symbols_below_threshold.push(c"__llvm_profile_counter_bias".to_owned());
 
     // If we're performing LTO for the entire crate graph, then for each of our
     // upstream dependencies, find the corresponding rlib and load the bitcode
@@ -84,37 +55,7 @@ fn prepare_lto(
     // with either fat or thin LTO
     let mut upstream_modules = Vec::new();
     if cgcx.lto != Lto::ThinLocal {
-        // Make sure we actually can run LTO
-        for crate_type in cgcx.crate_types.iter() {
-            if !crate_type_allows_lto(*crate_type) {
-                dcx.emit_err(LtoDisallowed);
-                return Err(FatalError);
-            } else if *crate_type == CrateType::Dylib {
-                if !cgcx.opts.unstable_opts.dylib_lto {
-                    dcx.emit_err(LtoDylib);
-                    return Err(FatalError);
-                }
-            } else if *crate_type == CrateType::ProcMacro && !cgcx.opts.unstable_opts.dylib_lto {
-                dcx.emit_err(LtoProcMacro);
-                return Err(FatalError);
-            }
-        }
-
-        if cgcx.opts.cg.prefer_dynamic && !cgcx.opts.unstable_opts.dylib_lto {
-            dcx.emit_err(DynamicLinkingWithLTO);
-            return Err(FatalError);
-        }
-
-        for &(cnum, ref path) in cgcx.each_linked_rlib_for_lto.iter() {
-            let exported_symbols =
-                cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO");
-            {
-                let _timer =
-                    cgcx.prof.generic_activity("LLVM_lto_generate_symbols_below_threshold");
-                symbols_below_threshold
-                    .extend(exported_symbols[&cnum].iter().filter_map(symbol_filter));
-            }
-
+        for path in each_linked_rlib_for_lto {
             let archive_data = unsafe {
                 Mmap::map(std::fs::File::open(&path).expect("couldn't open rlib"))
                     .expect("couldn't map rlib")
@@ -147,10 +88,6 @@ fn prepare_lto(
         }
     }
 
-    // __llvm_profile_counter_bias is pulled in at link time by an undefined reference to
-    // __llvm_profile_runtime, therefore we won't know until link time if this symbol
-    // should have default visibility.
-    symbols_below_threshold.push(c"__llvm_profile_counter_bias".to_owned());
     Ok((symbols_below_threshold, upstream_modules))
 }
 
@@ -199,15 +136,17 @@ fn get_bitcode_slice_from_object_data<'a>(
 /// for further optimization.
 pub(crate) fn run_fat(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
+    exported_symbols_for_lto: &[String],
+    each_linked_rlib_for_lto: &[PathBuf],
     modules: Vec<FatLtoInput<LlvmCodegenBackend>>,
-    cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
 ) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> {
     let dcx = cgcx.create_dcx();
     let dcx = dcx.handle();
-    let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, dcx)?;
+    let (symbols_below_threshold, upstream_modules) =
+        prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx)?;
     let symbols_below_threshold =
         symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();
-    fat_lto(cgcx, dcx, modules, cached_modules, upstream_modules, &symbols_below_threshold)
+    fat_lto(cgcx, dcx, modules, upstream_modules, &symbols_below_threshold)
 }
 
 /// Performs thin LTO by performing necessary global analysis and returning two
@@ -215,12 +154,15 @@ pub(crate) fn run_fat(
 /// can simply be copied over from the incr. comp. cache.
 pub(crate) fn run_thin(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
+    exported_symbols_for_lto: &[String],
+    each_linked_rlib_for_lto: &[PathBuf],
     modules: Vec<(String, ThinBuffer)>,
     cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
 ) -> Result<(Vec<ThinModule<LlvmCodegenBackend>>, Vec<WorkProduct>), FatalError> {
     let dcx = cgcx.create_dcx();
     let dcx = dcx.handle();
-    let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, dcx)?;
+    let (symbols_below_threshold, upstream_modules) =
+        prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx)?;
     let symbols_below_threshold =
         symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();
     if cgcx.opts.cg.linker_plugin_lto.enabled() {
@@ -245,7 +187,6 @@ fn fat_lto(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
     dcx: DiagCtxtHandle<'_>,
     modules: Vec<FatLtoInput<LlvmCodegenBackend>>,
-    cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
     mut serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>,
     symbols_below_threshold: &[*const libc::c_char],
 ) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> {
@@ -258,21 +199,12 @@ fn fat_lto(
     //   modules that are serialized in-memory.
     // * `in_memory` contains modules which are already parsed and in-memory,
     //   such as from multi-CGU builds.
-    //
-    // All of `cached_modules` (cached from previous incremental builds) can
-    // immediately go onto the `serialized_modules` modules list and then we can
-    // split the `modules` array into these two lists.
     let mut in_memory = Vec::new();
-    serialized_modules.extend(cached_modules.into_iter().map(|(buffer, wp)| {
-        info!("pushing cached module {:?}", wp.cgu_name);
-        (buffer, CString::new(wp.cgu_name).unwrap())
-    }));
     for module in modules {
         match module {
             FatLtoInput::InMemory(m) => in_memory.push(m),
             FatLtoInput::Serialized { name, buffer } => {
                 info!("pushing serialized module {:?}", name);
-                let buffer = SerializedModule::Local(buffer);
                 serialized_modules.push((buffer, CString::new(name).unwrap()));
             }
         }
diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs
index 574463be7ff..39a59560c9d 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs
@@ -39,10 +39,7 @@ impl Coords {
 /// or other expansions), and if it does happen then skipping a span or function is
 /// better than an ICE or `llvm-cov` failure that the user might have no way to avoid.
 pub(crate) fn make_coords(source_map: &SourceMap, file: &SourceFile, span: Span) -> Option<Coords> {
-    if span.is_empty() {
-        debug_assert!(false, "can't make coords from empty span: {span:?}");
-        return None;
-    }
+    let span = ensure_non_empty_span(source_map, span)?;
 
     let lo = span.lo();
     let hi = span.hi();
@@ -73,6 +70,29 @@ pub(crate) fn make_coords(source_map: &SourceMap, file: &SourceFile, span: Span)
     })
 }
 
+fn ensure_non_empty_span(source_map: &SourceMap, span: Span) -> Option<Span> {
+    if !span.is_empty() {
+        return Some(span);
+    }
+
+    // The span is empty, so try to enlarge it to cover an adjacent '{' or '}'.
+    source_map
+        .span_to_source(span, |src, start, end| try {
+            // Adjusting span endpoints by `BytePos(1)` is normally a bug,
+            // but in this case we have specifically checked that the character
+            // we're skipping over is one of two specific ASCII characters, so
+            // adjusting by exactly 1 byte is correct.
+            if src.as_bytes().get(end).copied() == Some(b'{') {
+                Some(span.with_hi(span.hi() + BytePos(1)))
+            } else if start > 0 && src.as_bytes()[start - 1] == b'}' {
+                Some(span.with_lo(span.lo() - BytePos(1)))
+            } else {
+                None
+            }
+        })
+        .ok()?
+}
+
 /// If `llvm-cov` sees a source region that is improperly ordered (end < start),
 /// it will immediately exit with a fatal error. To prevent that from happening,
 /// discard regions that are improperly ordered, or might be interpreted in a
diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs
index 31d49e86319..2a889888a39 100644
--- a/compiler/rustc_codegen_llvm/src/errors.rs
+++ b/compiler/rustc_codegen_llvm/src/errors.rs
@@ -20,11 +20,6 @@ pub(crate) struct SymbolAlreadyDefined<'a> {
 #[diag(codegen_llvm_sanitizer_memtag_requires_mte)]
 pub(crate) struct SanitizerMemtagRequiresMte;
 
-#[derive(Diagnostic)]
-#[diag(codegen_llvm_dynamic_linking_with_lto)]
-#[note]
-pub(crate) struct DynamicLinkingWithLTO;
-
 pub(crate) struct ParseTargetMachineConfig<'a>(pub LlvmError<'a>);
 
 impl<G: EmissionGuarantee> Diagnostic<'_, G> for ParseTargetMachineConfig<'_> {
@@ -42,18 +37,6 @@ impl<G: EmissionGuarantee> Diagnostic<'_, G> for ParseTargetMachineConfig<'_> {
 pub(crate) struct AutoDiffWithoutEnable;
 
 #[derive(Diagnostic)]
-#[diag(codegen_llvm_lto_disallowed)]
-pub(crate) struct LtoDisallowed;
-
-#[derive(Diagnostic)]
-#[diag(codegen_llvm_lto_dylib)]
-pub(crate) struct LtoDylib;
-
-#[derive(Diagnostic)]
-#[diag(codegen_llvm_lto_proc_macro)]
-pub(crate) struct LtoProcMacro;
-
-#[derive(Diagnostic)]
 #[diag(codegen_llvm_lto_bitcode_from_rlib)]
 pub(crate) struct LtoBitcodeFromRlib {
     pub llvm_err: String,
diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
index fcc0d378f06..7b27e496986 100644
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -382,26 +382,16 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                 let width = size.bits();
                 let llty = self.type_ix(width);
                 match name {
-                    sym::ctlz | sym::cttz => {
-                        let y = self.const_bool(false);
-                        let ret = self.call_intrinsic(
-                            format!("llvm.{name}"),
-                            &[llty],
-                            &[args[0].immediate(), y],
-                        );
-
-                        self.intcast(ret, result.layout.llvm_type(self), false)
-                    }
-                    sym::ctlz_nonzero => {
-                        let y = self.const_bool(true);
-                        let ret =
-                            self.call_intrinsic("llvm.ctlz", &[llty], &[args[0].immediate(), y]);
-                        self.intcast(ret, result.layout.llvm_type(self), false)
-                    }
-                    sym::cttz_nonzero => {
-                        let y = self.const_bool(true);
+                    sym::ctlz | sym::ctlz_nonzero | sym::cttz | sym::cttz_nonzero => {
+                        let y =
+                            self.const_bool(name == sym::ctlz_nonzero || name == sym::cttz_nonzero);
+                        let llvm_name = if name == sym::ctlz || name == sym::ctlz_nonzero {
+                            "llvm.ctlz"
+                        } else {
+                            "llvm.cttz"
+                        };
                         let ret =
-                            self.call_intrinsic("llvm.cttz", &[llty], &[args[0].immediate(), y]);
+                            self.call_intrinsic(llvm_name, &[llty], &[args[0].immediate(), y]);
                         self.intcast(ret, result.layout.llvm_type(self), false)
                     }
                     sym::ctpop => {
diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs
index aaf21f9ada9..8b1913cfa75 100644
--- a/compiler/rustc_codegen_llvm/src/lib.rs
+++ b/compiler/rustc_codegen_llvm/src/lib.rs
@@ -22,6 +22,7 @@
 use std::any::Any;
 use std::ffi::CStr;
 use std::mem::ManuallyDrop;
+use std::path::PathBuf;
 
 use back::owned_target_machine::OwnedTargetMachine;
 use back::write::{create_informational_target_machine, create_target_machine};
@@ -176,11 +177,13 @@ impl WriteBackendMethods for LlvmCodegenBackend {
     }
     fn run_and_optimize_fat_lto(
         cgcx: &CodegenContext<Self>,
+        exported_symbols_for_lto: &[String],
+        each_linked_rlib_for_lto: &[PathBuf],
         modules: Vec<FatLtoInput<Self>>,
-        cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>,
         diff_fncs: Vec<AutoDiffItem>,
     ) -> Result<ModuleCodegen<Self::Module>, FatalError> {
-        let mut module = back::lto::run_fat(cgcx, modules, cached_modules)?;
+        let mut module =
+            back::lto::run_fat(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, modules)?;
 
         if !diff_fncs.is_empty() {
             builder::autodiff::differentiate(&module, cgcx, diff_fncs)?;
@@ -194,10 +197,18 @@ impl WriteBackendMethods for LlvmCodegenBackend {
     }
     fn run_thin_lto(
         cgcx: &CodegenContext<Self>,
+        exported_symbols_for_lto: &[String],
+        each_linked_rlib_for_lto: &[PathBuf],
         modules: Vec<(String, Self::ThinBuffer)>,
         cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>,
     ) -> Result<(Vec<ThinModule<Self>>, Vec<WorkProduct>), FatalError> {
-        back::lto::run_thin(cgcx, modules, cached_modules)
+        back::lto::run_thin(
+            cgcx,
+            exported_symbols_for_lto,
+            each_linked_rlib_for_lto,
+            modules,
+            cached_modules,
+        )
     }
     fn optimize(
         cgcx: &CodegenContext<Self>,
diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs
index 0fb987bdf82..53899da183a 100644
--- a/compiler/rustc_codegen_llvm/src/llvm_util.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs
@@ -405,6 +405,8 @@ fn update_target_reliable_float_cfg(sess: &Session, cfg: &mut TargetConfig) {
         ("mips64" | "mips64r6", _) => false,
         // Selection bug <https://github.com/llvm/llvm-project/issues/95471>
         ("nvptx64", _) => false,
+        // Unsupported https://github.com/llvm/llvm-project/issues/121122
+        ("amdgpu", _) => false,
         // ABI bugs <https://github.com/rust-lang/rust/issues/125109> et al. (full
         // list at <https://github.com/rust-lang/rust/issues/116909>)
         ("powerpc" | "powerpc64", _) => false,
@@ -433,6 +435,9 @@ fn update_target_reliable_float_cfg(sess: &Session, cfg: &mut TargetConfig) {
         // This rules out anything that doesn't have `long double` = `binary128`; <= 32 bits
         // (ld is `f64`), anything other than Linux (Windows and MacOS use `f64`), and `x86`
         // (ld is 80-bit extended precision).
+        //
+        // musl does not implement the symbols required for f128 math at all.
+        _ if target_env == "musl" => false,
         ("x86_64", _) => false,
         (_, "linux") if target_pointer_width == 64 => true,
         _ => false,
diff --git a/compiler/rustc_codegen_ssa/messages.ftl b/compiler/rustc_codegen_ssa/messages.ftl
index c7bd6ffd1f2..a70d0011d16 100644
--- a/compiler/rustc_codegen_ssa/messages.ftl
+++ b/compiler/rustc_codegen_ssa/messages.ftl
@@ -35,6 +35,10 @@ codegen_ssa_dlltool_fail_import_library =
     {$stdout}
     {$stderr}
 
+codegen_ssa_dynamic_linking_with_lto =
+    cannot prefer dynamic linking when performing LTO
+    .note = only 'staticlib', 'bin', and 'cdylib' outputs are supported with LTO
+
 codegen_ssa_error_calling_dlltool =
     Error calling dlltool '{$dlltool_path}': {$error}
 
@@ -191,6 +195,12 @@ codegen_ssa_linker_unsupported_modifier = `as-needed` modifier not supported for
 
 codegen_ssa_linking_failed = linking with `{$linker_path}` failed: {$exit_status}
 
+codegen_ssa_lto_disallowed = lto can only be run for executables, cdylibs and static library outputs
+
+codegen_ssa_lto_dylib = lto cannot be used for `dylib` crate type without `-Zdylib-lto`
+
+codegen_ssa_lto_proc_macro = lto cannot be used for `proc-macro` crate type without `-Zdylib-lto`
+
 codegen_ssa_malformed_cgu_name =
     found malformed codegen unit name `{$user_path}`. codegen units names must always start with the name of the crate (`{$crate_name}` in this case).
 
diff --git a/compiler/rustc_codegen_ssa/src/back/link/raw_dylib.rs b/compiler/rustc_codegen_ssa/src/back/link/raw_dylib.rs
index 74f39022afb..b9e0c957363 100644
--- a/compiler/rustc_codegen_ssa/src/back/link/raw_dylib.rs
+++ b/compiler/rustc_codegen_ssa/src/back/link/raw_dylib.rs
@@ -4,7 +4,7 @@ use std::path::{Path, PathBuf};
 
 use rustc_abi::Endian;
 use rustc_data_structures::base_n::{CASE_INSENSITIVE, ToBaseN};
-use rustc_data_structures::fx::FxIndexMap;
+use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
 use rustc_data_structures::stable_hasher::StableHasher;
 use rustc_hashes::Hash128;
 use rustc_session::Session;
@@ -214,7 +214,7 @@ pub(super) fn create_raw_dylib_elf_stub_shared_objects<'a>(
 /// It exports all the provided symbols, but is otherwise empty.
 fn create_elf_raw_dylib_stub(sess: &Session, soname: &str, symbols: &[DllImport]) -> Vec<u8> {
     use object::write::elf as write;
-    use object::{Architecture, elf};
+    use object::{AddressSize, Architecture, elf};
 
     let mut stub_buf = Vec::new();
 
@@ -226,54 +226,94 @@ fn create_elf_raw_dylib_stub(sess: &Session, soname: &str, symbols: &[DllImport]
     // It is important that the order of reservation matches the order of writing.
     // The object crate contains many debug asserts that fire if you get this wrong.
 
+    let Some((arch, sub_arch)) = sess.target.object_architecture(&sess.unstable_target_features)
+    else {
+        sess.dcx().fatal(format!(
+            "raw-dylib is not supported for the architecture `{}`",
+            sess.target.arch
+        ));
+    };
+
     let endianness = match sess.target.options.endian {
         Endian::Little => object::Endianness::Little,
         Endian::Big => object::Endianness::Big,
     };
-    let mut stub = write::Writer::new(endianness, true, &mut stub_buf);
+
+    let is_64 = match arch.address_size() {
+        Some(AddressSize::U8 | AddressSize::U16 | AddressSize::U32) => false,
+        Some(AddressSize::U64) => true,
+        _ => sess.dcx().fatal(format!(
+            "raw-dylib is not supported for the architecture `{}`",
+            sess.target.arch
+        )),
+    };
+
+    let mut stub = write::Writer::new(endianness, is_64, &mut stub_buf);
+
+    let mut vers = Vec::new();
+    let mut vers_map = FxHashMap::default();
+    let mut syms = Vec::new();
+
+    for symbol in symbols {
+        let symbol_name = symbol.name.as_str();
+        if let Some((name, version_name)) = symbol_name.split_once('@') {
+            assert!(!version_name.contains('@'));
+            let dynstr = stub.add_dynamic_string(name.as_bytes());
+            let ver = if let Some(&ver_id) = vers_map.get(version_name) {
+                ver_id
+            } else {
+                let id = vers.len();
+                vers_map.insert(version_name, id);
+                let dynstr = stub.add_dynamic_string(version_name.as_bytes());
+                vers.push((version_name, dynstr));
+                id
+            };
+            syms.push((name, dynstr, Some(ver)));
+        } else {
+            let dynstr = stub.add_dynamic_string(symbol_name.as_bytes());
+            syms.push((symbol_name, dynstr, None));
+        }
+    }
+
+    let soname = stub.add_dynamic_string(soname.as_bytes());
 
     // These initial reservations don't reserve any bytes in the binary yet,
     // they just allocate in the internal data structures.
 
-    // First, we crate the dynamic symbol table. It starts with a null symbol
+    // First, we create the dynamic symbol table. It starts with a null symbol
     // and then all the symbols and their dynamic strings.
     stub.reserve_null_dynamic_symbol_index();
 
-    let dynstrs = symbols
-        .iter()
-        .map(|sym| {
-            stub.reserve_dynamic_symbol_index();
-            (sym, stub.add_dynamic_string(sym.name.as_str().as_bytes()))
-        })
-        .collect::<Vec<_>>();
-
-    let soname = stub.add_dynamic_string(soname.as_bytes());
+    for _ in syms.iter() {
+        stub.reserve_dynamic_symbol_index();
+    }
 
     // Reserve the sections.
     // We have the minimal sections for a dynamic SO and .text where we point our dummy symbols to.
     stub.reserve_shstrtab_section_index();
     let text_section_name = stub.add_section_name(".text".as_bytes());
     let text_section = stub.reserve_section_index();
-    stub.reserve_dynstr_section_index();
     stub.reserve_dynsym_section_index();
+    stub.reserve_dynstr_section_index();
+    if !vers.is_empty() {
+        stub.reserve_gnu_versym_section_index();
+        stub.reserve_gnu_verdef_section_index();
+    }
     stub.reserve_dynamic_section_index();
 
     // These reservations now determine the actual layout order of the object file.
     stub.reserve_file_header();
     stub.reserve_shstrtab();
     stub.reserve_section_headers();
-    stub.reserve_dynstr();
     stub.reserve_dynsym();
+    stub.reserve_dynstr();
+    if !vers.is_empty() {
+        stub.reserve_gnu_versym();
+        stub.reserve_gnu_verdef(1 + vers.len(), 1 + vers.len());
+    }
     stub.reserve_dynamic(2); // DT_SONAME, DT_NULL
 
     // First write the ELF header with the arch information.
-    let Some((arch, sub_arch)) = sess.target.object_architecture(&sess.unstable_target_features)
-    else {
-        sess.dcx().fatal(format!(
-            "raw-dylib is not supported for the architecture `{}`",
-            sess.target.arch
-        ));
-    };
     let e_machine = match (arch, sub_arch) {
         (Architecture::Aarch64, None) => elf::EM_AARCH64,
         (Architecture::Aarch64_Ilp32, None) => elf::EM_AARCH64,
@@ -342,18 +382,19 @@ fn create_elf_raw_dylib_stub(sess: &Session, soname: &str, symbols: &[DllImport]
         sh_addralign: 1,
         sh_entsize: 0,
     });
-    stub.write_dynstr_section_header(0);
     stub.write_dynsym_section_header(0, 1);
+    stub.write_dynstr_section_header(0);
+    if !vers.is_empty() {
+        stub.write_gnu_versym_section_header(0);
+        stub.write_gnu_verdef_section_header(0);
+    }
     stub.write_dynamic_section_header(0);
 
-    // .dynstr
-    stub.write_dynstr();
-
     // .dynsym
     stub.write_null_dynamic_symbol();
-    for (_, name) in dynstrs {
+    for (_name, dynstr, _ver) in syms.iter().copied() {
         stub.write_dynamic_symbol(&write::Sym {
-            name: Some(name),
+            name: Some(dynstr),
             st_info: (elf::STB_GLOBAL << 4) | elf::STT_NOTYPE,
             st_other: elf::STV_DEFAULT,
             section: Some(text_section),
@@ -363,10 +404,47 @@ fn create_elf_raw_dylib_stub(sess: &Session, soname: &str, symbols: &[DllImport]
         });
     }
 
+    // .dynstr
+    stub.write_dynstr();
+
+    // ld.bfd is unhappy if these sections exist without any symbols, so we only generate them when necessary.
+    if !vers.is_empty() {
+        // .gnu_version
+        stub.write_null_gnu_versym();
+        for (_name, _dynstr, ver) in syms.iter().copied() {
+            stub.write_gnu_versym(if let Some(ver) = ver {
+                assert!((2 + ver as u16) < elf::VERSYM_HIDDEN);
+                elf::VERSYM_HIDDEN | (2 + ver as u16)
+            } else {
+                1
+            });
+        }
+
+        // .gnu_version_d
+        stub.write_align_gnu_verdef();
+        stub.write_gnu_verdef(&write::Verdef {
+            version: elf::VER_DEF_CURRENT,
+            flags: elf::VER_FLG_BASE,
+            index: 1,
+            aux_count: 1,
+            name: soname,
+        });
+        for (ver, (_name, dynstr)) in vers.into_iter().enumerate() {
+            stub.write_gnu_verdef(&write::Verdef {
+                version: elf::VER_DEF_CURRENT,
+                flags: 0,
+                index: 2 + ver as u16,
+                aux_count: 1,
+                name: dynstr,
+            });
+        }
+    }
+
     // .dynamic
     // the DT_SONAME will be used by the linker to populate DT_NEEDED
     // which the loader uses to find the library.
     // DT_NULL terminates the .dynamic table.
+    stub.write_align_dynamic();
     stub.write_dynamic_string(elf::DT_SONAME, soname);
     stub.write_dynamic(elf::DT_NULL, 0);
 
diff --git a/compiler/rustc_codegen_ssa/src/back/lto.rs b/compiler/rustc_codegen_ssa/src/back/lto.rs
index b49b6783bbd..c95038375a1 100644
--- a/compiler/rustc_codegen_ssa/src/back/lto.rs
+++ b/compiler/rustc_codegen_ssa/src/back/lto.rs
@@ -2,7 +2,15 @@ use std::ffi::CString;
 use std::sync::Arc;
 
 use rustc_data_structures::memmap::Mmap;
+use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
+use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo, SymbolExportLevel};
+use rustc_middle::ty::TyCtxt;
+use rustc_session::config::{CrateType, Lto};
+use tracing::info;
 
+use crate::back::symbol_export::{self, symbol_name_for_instance_in_crate};
+use crate::back::write::CodegenContext;
+use crate::errors::{DynamicLinkingWithLTO, LtoDisallowed, LtoDylib, LtoProcMacro};
 use crate::traits::*;
 
 pub struct ThinModule<B: WriteBackendMethods> {
@@ -52,3 +60,86 @@ impl<M: ModuleBufferMethods> SerializedModule<M> {
         }
     }
 }
+
+fn crate_type_allows_lto(crate_type: CrateType) -> bool {
+    match crate_type {
+        CrateType::Executable
+        | CrateType::Dylib
+        | CrateType::Staticlib
+        | CrateType::Cdylib
+        | CrateType::ProcMacro
+        | CrateType::Sdylib => true,
+        CrateType::Rlib => false,
+    }
+}
+
+pub(super) fn exported_symbols_for_lto(
+    tcx: TyCtxt<'_>,
+    each_linked_rlib_for_lto: &[CrateNum],
+) -> Vec<String> {
+    let export_threshold = match tcx.sess.lto() {
+        // We're just doing LTO for our one crate
+        Lto::ThinLocal => SymbolExportLevel::Rust,
+
+        // We're doing LTO for the entire crate graph
+        Lto::Fat | Lto::Thin => symbol_export::crates_export_threshold(&tcx.crate_types()),
+
+        Lto::No => return vec![],
+    };
+
+    let copy_symbols = |cnum| {
+        tcx.exported_non_generic_symbols(cnum)
+            .iter()
+            .chain(tcx.exported_generic_symbols(cnum))
+            .filter_map(|&(s, info): &(ExportedSymbol<'_>, SymbolExportInfo)| {
+                if info.level.is_below_threshold(export_threshold) || info.used {
+                    Some(symbol_name_for_instance_in_crate(tcx, s, cnum))
+                } else {
+                    None
+                }
+            })
+            .collect::<Vec<_>>()
+    };
+    let mut symbols_below_threshold = {
+        let _timer = tcx.prof.generic_activity("lto_generate_symbols_below_threshold");
+        copy_symbols(LOCAL_CRATE)
+    };
+    info!("{} symbols to preserve in this crate", symbols_below_threshold.len());
+
+    // If we're performing LTO for the entire crate graph, then for each of our
+    // upstream dependencies, include their exported symbols.
+    if tcx.sess.lto() != Lto::ThinLocal {
+        for &cnum in each_linked_rlib_for_lto {
+            let _timer = tcx.prof.generic_activity("lto_generate_symbols_below_threshold");
+            symbols_below_threshold.extend(copy_symbols(cnum));
+        }
+    }
+
+    symbols_below_threshold
+}
+
+pub(super) fn check_lto_allowed<B: WriteBackendMethods>(cgcx: &CodegenContext<B>) {
+    if cgcx.lto == Lto::ThinLocal {
+        // Crate local LTO is always allowed
+        return;
+    }
+
+    let dcx = cgcx.create_dcx();
+
+    // Make sure we actually can run LTO
+    for crate_type in cgcx.crate_types.iter() {
+        if !crate_type_allows_lto(*crate_type) {
+            dcx.handle().emit_fatal(LtoDisallowed);
+        } else if *crate_type == CrateType::Dylib {
+            if !cgcx.opts.unstable_opts.dylib_lto {
+                dcx.handle().emit_fatal(LtoDylib);
+            }
+        } else if *crate_type == CrateType::ProcMacro && !cgcx.opts.unstable_opts.dylib_lto {
+            dcx.handle().emit_fatal(LtoProcMacro);
+        }
+    }
+
+    if cgcx.opts.cg.prefer_dynamic && !cgcx.opts.unstable_opts.dylib_lto {
+        dcx.handle().emit_fatal(DynamicLinkingWithLTO);
+    }
+}
diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs
index 24e0a4eb533..7be274df1d4 100644
--- a/compiler/rustc_codegen_ssa/src/back/write.rs
+++ b/compiler/rustc_codegen_ssa/src/back/write.rs
@@ -9,7 +9,7 @@ use std::{fs, io, mem, str, thread};
 use rustc_abi::Size;
 use rustc_ast::attr;
 use rustc_ast::expand::autodiff_attrs::AutoDiffItem;
-use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
+use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::jobserver::{self, Acquired};
 use rustc_data_structures::memmap::Mmap;
 use rustc_data_structures::profiling::{SelfProfilerRef, VerboseTimingGuard};
@@ -20,14 +20,12 @@ use rustc_errors::{
     Suggestions,
 };
 use rustc_fs_util::link_or_copy;
-use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
 use rustc_incremental::{
     copy_cgu_workproduct_to_incr_comp_cache_dir, in_incr_comp_dir, in_incr_comp_dir_sess,
 };
 use rustc_metadata::fs::copy_to_stdout;
 use rustc_middle::bug;
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
-use rustc_middle::middle::exported_symbols::SymbolExportInfo;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::Session;
 use rustc_session::config::{
@@ -40,7 +38,7 @@ use tracing::debug;
 
 use super::link::{self, ensure_removed};
 use super::lto::{self, SerializedModule};
-use super::symbol_export::symbol_name_for_instance_in_crate;
+use crate::back::lto::check_lto_allowed;
 use crate::errors::{AutodiffWithoutLto, ErrorCreatingRemarkDir};
 use crate::traits::*;
 use crate::{
@@ -332,8 +330,6 @@ pub type TargetMachineFactoryFn<B> = Arc<
         + Sync,
 >;
 
-type ExportedSymbols = FxHashMap<CrateNum, Arc<Vec<(String, SymbolExportInfo)>>>;
-
 /// Additional resources used by optimize_and_codegen (not module specific)
 #[derive(Clone)]
 pub struct CodegenContext<B: WriteBackendMethods> {
@@ -343,10 +339,8 @@ pub struct CodegenContext<B: WriteBackendMethods> {
     pub save_temps: bool,
     pub fewer_names: bool,
     pub time_trace: bool,
-    pub exported_symbols: Option<Arc<ExportedSymbols>>,
     pub opts: Arc<config::Options>,
     pub crate_types: Vec<CrateType>,
-    pub each_linked_rlib_for_lto: Vec<(CrateNum, PathBuf)>,
     pub output_filenames: Arc<OutputFilenames>,
     pub invocation_temp: Option<String>,
     pub regular_module_config: Arc<ModuleConfig>,
@@ -401,13 +395,21 @@ impl<B: WriteBackendMethods> CodegenContext<B> {
 
 fn generate_thin_lto_work<B: ExtraBackendMethods>(
     cgcx: &CodegenContext<B>,
+    exported_symbols_for_lto: &[String],
+    each_linked_rlib_for_lto: &[PathBuf],
     needs_thin_lto: Vec<(String, B::ThinBuffer)>,
     import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>,
 ) -> Vec<(WorkItem<B>, u64)> {
     let _prof_timer = cgcx.prof.generic_activity("codegen_thin_generate_lto_work");
 
-    let (lto_modules, copy_jobs) =
-        B::run_thin_lto(cgcx, needs_thin_lto, import_only_modules).unwrap_or_else(|e| e.raise());
+    let (lto_modules, copy_jobs) = B::run_thin_lto(
+        cgcx,
+        exported_symbols_for_lto,
+        each_linked_rlib_for_lto,
+        needs_thin_lto,
+        import_only_modules,
+    )
+    .unwrap_or_else(|e| e.raise());
     lto_modules
         .into_iter()
         .map(|module| {
@@ -723,6 +725,8 @@ pub(crate) enum WorkItem<B: WriteBackendMethods> {
     CopyPostLtoArtifacts(CachedModuleCodegen),
     /// Performs fat LTO on the given module.
     FatLto {
+        exported_symbols_for_lto: Arc<Vec<String>>,
+        each_linked_rlib_for_lto: Vec<PathBuf>,
         needs_fat_lto: Vec<FatLtoInput<B>>,
         import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>,
         autodiff: Vec<AutoDiffItem>,
@@ -810,7 +814,7 @@ pub(crate) enum WorkItemResult<B: WriteBackendMethods> {
 }
 
 pub enum FatLtoInput<B: WriteBackendMethods> {
-    Serialized { name: String, buffer: B::ModuleBuffer },
+    Serialized { name: String, buffer: SerializedModule<B::ModuleBuffer> },
     InMemory(ModuleCodegen<B::Module>),
 }
 
@@ -899,7 +903,10 @@ fn execute_optimize_work_item<B: ExtraBackendMethods>(
                 fs::write(&path, buffer.data()).unwrap_or_else(|e| {
                     panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e);
                 });
-                Ok(WorkItemResult::NeedsFatLto(FatLtoInput::Serialized { name, buffer }))
+                Ok(WorkItemResult::NeedsFatLto(FatLtoInput::Serialized {
+                    name,
+                    buffer: SerializedModule::Local(buffer),
+                }))
             }
             None => Ok(WorkItemResult::NeedsFatLto(FatLtoInput::InMemory(module))),
         },
@@ -992,12 +999,24 @@ fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>(
 
 fn execute_fat_lto_work_item<B: ExtraBackendMethods>(
     cgcx: &CodegenContext<B>,
-    needs_fat_lto: Vec<FatLtoInput<B>>,
+    exported_symbols_for_lto: &[String],
+    each_linked_rlib_for_lto: &[PathBuf],
+    mut needs_fat_lto: Vec<FatLtoInput<B>>,
     import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>,
     autodiff: Vec<AutoDiffItem>,
     module_config: &ModuleConfig,
 ) -> Result<WorkItemResult<B>, FatalError> {
-    let module = B::run_and_optimize_fat_lto(cgcx, needs_fat_lto, import_only_modules, autodiff)?;
+    for (module, wp) in import_only_modules {
+        needs_fat_lto.push(FatLtoInput::Serialized { name: wp.cgu_name, buffer: module })
+    }
+
+    let module = B::run_and_optimize_fat_lto(
+        cgcx,
+        exported_symbols_for_lto,
+        each_linked_rlib_for_lto,
+        needs_fat_lto,
+        autodiff,
+    )?;
     let module = B::codegen(cgcx, module, module_config)?;
     Ok(WorkItemResult::Finished(module))
 }
@@ -1032,7 +1051,7 @@ pub(crate) enum Message<B: WriteBackendMethods> {
 
     /// The backend has finished processing a work item for a codegen unit.
     /// Sent from a backend worker thread.
-    WorkItem { result: Result<WorkItemResult<B>, Option<WorkerFatalError>>, worker_id: usize },
+    WorkItem { result: Result<WorkItemResult<B>, Option<WorkerFatalError>> },
 
     /// The frontend has finished generating something (backend IR or a
     /// post-LTO artifact) for a codegen unit, and it should be passed to the
@@ -1113,42 +1132,18 @@ fn start_executing_work<B: ExtraBackendMethods>(
     let autodiff_items = autodiff_items.to_vec();
 
     let mut each_linked_rlib_for_lto = Vec::new();
+    let mut each_linked_rlib_file_for_lto = Vec::new();
     drop(link::each_linked_rlib(crate_info, None, &mut |cnum, path| {
         if link::ignored_for_lto(sess, crate_info, cnum) {
             return;
         }
-        each_linked_rlib_for_lto.push((cnum, path.to_path_buf()));
+        each_linked_rlib_for_lto.push(cnum);
+        each_linked_rlib_file_for_lto.push(path.to_path_buf());
     }));
 
     // Compute the set of symbols we need to retain when doing LTO (if we need to)
-    let exported_symbols = {
-        let mut exported_symbols = FxHashMap::default();
-
-        let copy_symbols = |cnum| {
-            let symbols = tcx
-                .exported_non_generic_symbols(cnum)
-                .iter()
-                .chain(tcx.exported_generic_symbols(cnum))
-                .map(|&(s, lvl)| (symbol_name_for_instance_in_crate(tcx, s, cnum), lvl))
-                .collect();
-            Arc::new(symbols)
-        };
-
-        match sess.lto() {
-            Lto::No => None,
-            Lto::ThinLocal => {
-                exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
-                Some(Arc::new(exported_symbols))
-            }
-            Lto::Fat | Lto::Thin => {
-                exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
-                for &(cnum, ref _path) in &each_linked_rlib_for_lto {
-                    exported_symbols.insert(cnum, copy_symbols(cnum));
-                }
-                Some(Arc::new(exported_symbols))
-            }
-        }
-    };
+    let exported_symbols_for_lto =
+        Arc::new(lto::exported_symbols_for_lto(tcx, &each_linked_rlib_for_lto));
 
     // First up, convert our jobserver into a helper thread so we can use normal
     // mpsc channels to manage our messages and such.
@@ -1183,14 +1178,12 @@ fn start_executing_work<B: ExtraBackendMethods>(
 
     let cgcx = CodegenContext::<B> {
         crate_types: tcx.crate_types().to_vec(),
-        each_linked_rlib_for_lto,
         lto: sess.lto(),
         fewer_names: sess.fewer_names(),
         save_temps: sess.opts.cg.save_temps,
         time_trace: sess.opts.unstable_opts.llvm_time_trace,
         opts: Arc::new(sess.opts.clone()),
         prof: sess.prof.clone(),
-        exported_symbols,
         remark: sess.opts.cg.remark.clone(),
         remark_dir,
         incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()),
@@ -1350,18 +1343,6 @@ fn start_executing_work<B: ExtraBackendMethods>(
     // necessary. There's already optimizations in place to avoid sending work
     // back to the coordinator if LTO isn't requested.
     return B::spawn_named_thread(cgcx.time_trace, "coordinator".to_string(), move || {
-        let mut worker_id_counter = 0;
-        let mut free_worker_ids = Vec::new();
-        let mut get_worker_id = |free_worker_ids: &mut Vec<usize>| {
-            if let Some(id) = free_worker_ids.pop() {
-                id
-            } else {
-                let id = worker_id_counter;
-                worker_id_counter += 1;
-                id
-            }
-        };
-
         // This is where we collect codegen units that have gone all the way
         // through codegen and LLVM.
         let mut compiled_modules = vec![];
@@ -1442,12 +1423,7 @@ fn start_executing_work<B: ExtraBackendMethods>(
                         let (item, _) =
                             work_items.pop().expect("queue empty - queue_full_enough() broken?");
                         main_thread_state = MainThreadState::Lending;
-                        spawn_work(
-                            &cgcx,
-                            &mut llvm_start_time,
-                            get_worker_id(&mut free_worker_ids),
-                            item,
-                        );
+                        spawn_work(&cgcx, &mut llvm_start_time, item);
                     }
                 }
             } else if codegen_state == Completed {
@@ -1474,12 +1450,18 @@ fn start_executing_work<B: ExtraBackendMethods>(
                     let needs_fat_lto = mem::take(&mut needs_fat_lto);
                     let needs_thin_lto = mem::take(&mut needs_thin_lto);
                     let import_only_modules = mem::take(&mut lto_import_only_modules);
+                    let each_linked_rlib_file_for_lto =
+                        mem::take(&mut each_linked_rlib_file_for_lto);
+
+                    check_lto_allowed(&cgcx);
 
                     if !needs_fat_lto.is_empty() {
                         assert!(needs_thin_lto.is_empty());
 
                         work_items.push((
                             WorkItem::FatLto {
+                                exported_symbols_for_lto: Arc::clone(&exported_symbols_for_lto),
+                                each_linked_rlib_for_lto: each_linked_rlib_file_for_lto,
                                 needs_fat_lto,
                                 import_only_modules,
                                 autodiff: autodiff_items.clone(),
@@ -1495,9 +1477,13 @@ fn start_executing_work<B: ExtraBackendMethods>(
                             dcx.handle().emit_fatal(AutodiffWithoutLto {});
                         }
 
-                        for (work, cost) in
-                            generate_thin_lto_work(&cgcx, needs_thin_lto, import_only_modules)
-                        {
+                        for (work, cost) in generate_thin_lto_work(
+                            &cgcx,
+                            &exported_symbols_for_lto,
+                            &each_linked_rlib_file_for_lto,
+                            needs_thin_lto,
+                            import_only_modules,
+                        ) {
                             let insertion_index = work_items
                                 .binary_search_by_key(&cost, |&(_, cost)| cost)
                                 .unwrap_or_else(|e| e);
@@ -1516,12 +1502,7 @@ fn start_executing_work<B: ExtraBackendMethods>(
                     MainThreadState::Idle => {
                         if let Some((item, _)) = work_items.pop() {
                             main_thread_state = MainThreadState::Lending;
-                            spawn_work(
-                                &cgcx,
-                                &mut llvm_start_time,
-                                get_worker_id(&mut free_worker_ids),
-                                item,
-                            );
+                            spawn_work(&cgcx, &mut llvm_start_time, item);
                         } else {
                             // There is no unstarted work, so let the main thread
                             // take over for a running worker. Otherwise the
@@ -1557,12 +1538,7 @@ fn start_executing_work<B: ExtraBackendMethods>(
                 while running_with_own_token < tokens.len()
                     && let Some((item, _)) = work_items.pop()
                 {
-                    spawn_work(
-                        &cgcx,
-                        &mut llvm_start_time,
-                        get_worker_id(&mut free_worker_ids),
-                        item,
-                    );
+                    spawn_work(&cgcx, &mut llvm_start_time, item);
                     running_with_own_token += 1;
                 }
             }
@@ -1570,21 +1546,6 @@ fn start_executing_work<B: ExtraBackendMethods>(
             // Relinquish accidentally acquired extra tokens.
             tokens.truncate(running_with_own_token);
 
-            // If a thread exits successfully then we drop a token associated
-            // with that worker and update our `running_with_own_token` count.
-            // We may later re-acquire a token to continue running more work.
-            // We may also not actually drop a token here if the worker was
-            // running with an "ephemeral token".
-            let mut free_worker = |worker_id| {
-                if main_thread_state == MainThreadState::Lending {
-                    main_thread_state = MainThreadState::Idle;
-                } else {
-                    running_with_own_token -= 1;
-                }
-
-                free_worker_ids.push(worker_id);
-            };
-
             let msg = coordinator_receive.recv().unwrap();
             match *msg.downcast::<Message<B>>().ok().unwrap() {
                 // Save the token locally and the next turn of the loop will use
@@ -1653,8 +1614,17 @@ fn start_executing_work<B: ExtraBackendMethods>(
                     codegen_state = Aborted;
                 }
 
-                Message::WorkItem { result, worker_id } => {
-                    free_worker(worker_id);
+                Message::WorkItem { result } => {
+                    // If a thread exits successfully then we drop a token associated
+                    // with that worker and update our `running_with_own_token` count.
+                    // We may later re-acquire a token to continue running more work.
+                    // We may also not actually drop a token here if the worker was
+                    // running with an "ephemeral token".
+                    if main_thread_state == MainThreadState::Lending {
+                        main_thread_state = MainThreadState::Idle;
+                    } else {
+                        running_with_own_token -= 1;
+                    }
 
                     match result {
                         Ok(WorkItemResult::Finished(compiled_module)) => {
@@ -1800,7 +1770,6 @@ pub(crate) struct WorkerFatalError;
 fn spawn_work<'a, B: ExtraBackendMethods>(
     cgcx: &'a CodegenContext<B>,
     llvm_start_time: &mut Option<VerboseTimingGuard<'a>>,
-    worker_id: usize,
     work: WorkItem<B>,
 ) {
     if cgcx.config(work.module_kind()).time_module && llvm_start_time.is_none() {
@@ -1815,24 +1784,21 @@ fn spawn_work<'a, B: ExtraBackendMethods>(
         struct Bomb<B: ExtraBackendMethods> {
             coordinator_send: Sender<Box<dyn Any + Send>>,
             result: Option<Result<WorkItemResult<B>, FatalError>>,
-            worker_id: usize,
         }
         impl<B: ExtraBackendMethods> Drop for Bomb<B> {
             fn drop(&mut self) {
-                let worker_id = self.worker_id;
                 let msg = match self.result.take() {
-                    Some(Ok(result)) => Message::WorkItem::<B> { result: Ok(result), worker_id },
+                    Some(Ok(result)) => Message::WorkItem::<B> { result: Ok(result) },
                     Some(Err(FatalError)) => {
-                        Message::WorkItem::<B> { result: Err(Some(WorkerFatalError)), worker_id }
+                        Message::WorkItem::<B> { result: Err(Some(WorkerFatalError)) }
                     }
-                    None => Message::WorkItem::<B> { result: Err(None), worker_id },
+                    None => Message::WorkItem::<B> { result: Err(None) },
                 };
                 drop(self.coordinator_send.send(Box::new(msg)));
             }
         }
 
-        let mut bomb =
-            Bomb::<B> { coordinator_send: cgcx.coordinator_send.clone(), result: None, worker_id };
+        let mut bomb = Bomb::<B> { coordinator_send: cgcx.coordinator_send.clone(), result: None };
 
         // Execute the work itself, and if it finishes successfully then flag
         // ourselves as a success as well.
@@ -1856,12 +1822,20 @@ fn spawn_work<'a, B: ExtraBackendMethods>(
                     );
                     Ok(execute_copy_from_cache_work_item(&cgcx, m, module_config))
                 }
-                WorkItem::FatLto { needs_fat_lto, import_only_modules, autodiff } => {
+                WorkItem::FatLto {
+                    exported_symbols_for_lto,
+                    each_linked_rlib_for_lto,
+                    needs_fat_lto,
+                    import_only_modules,
+                    autodiff,
+                } => {
                     let _timer = cgcx
                         .prof
                         .generic_activity_with_arg("codegen_module_perform_lto", "everything");
                     execute_fat_lto_work_item(
                         &cgcx,
+                        &exported_symbols_for_lto,
+                        &each_linked_rlib_for_lto,
                         needs_fat_lto,
                         import_only_modules,
                         autodiff,
diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
index dd49db26689..3f456fa115a 100644
--- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
+++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
@@ -511,15 +511,6 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
         err.emit();
     }
 
-    // Any linkage to LLVM intrinsics for now forcibly marks them all as never
-    // unwinds since LLVM sometimes can't handle codegen which `invoke`s
-    // intrinsic functions.
-    if let Some(name) = &codegen_fn_attrs.link_name
-        && name.as_str().starts_with("llvm.")
-    {
-        codegen_fn_attrs.flags |= CodegenFnAttrFlags::NEVER_UNWIND;
-    }
-
     if let Some(features) = check_tied_features(
         tcx.sess,
         &codegen_fn_attrs
diff --git a/compiler/rustc_codegen_ssa/src/common.rs b/compiler/rustc_codegen_ssa/src/common.rs
index 48565e0b4de..a6fd6c763ed 100644
--- a/compiler/rustc_codegen_ssa/src/common.rs
+++ b/compiler/rustc_codegen_ssa/src/common.rs
@@ -148,7 +148,7 @@ pub(crate) fn shift_mask_val<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 pub fn asm_const_to_str<'tcx>(
     tcx: TyCtxt<'tcx>,
     sp: Span,
-    const_value: mir::ConstValue<'tcx>,
+    const_value: mir::ConstValue,
     ty_and_layout: TyAndLayout<'tcx>,
 ) -> String {
     let mir::ConstValue::Scalar(scalar) = const_value else {
diff --git a/compiler/rustc_codegen_ssa/src/errors.rs b/compiler/rustc_codegen_ssa/src/errors.rs
index 9040915b6af..3d787d8bdbd 100644
--- a/compiler/rustc_codegen_ssa/src/errors.rs
+++ b/compiler/rustc_codegen_ssa/src/errors.rs
@@ -1294,3 +1294,20 @@ pub(crate) struct FeatureNotValid<'a> {
     #[help]
     pub plus_hint: bool,
 }
+
+#[derive(Diagnostic)]
+#[diag(codegen_ssa_lto_disallowed)]
+pub(crate) struct LtoDisallowed;
+
+#[derive(Diagnostic)]
+#[diag(codegen_ssa_lto_dylib)]
+pub(crate) struct LtoDylib;
+
+#[derive(Diagnostic)]
+#[diag(codegen_ssa_lto_proc_macro)]
+pub(crate) struct LtoProcMacro;
+
+#[derive(Diagnostic)]
+#[diag(codegen_ssa_dynamic_linking_with_lto)]
+#[note]
+pub(crate) struct DynamicLinkingWithLTO;
diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
index 6d6465dd798..c2c023af090 100644
--- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
@@ -1,12 +1,13 @@
 //! An analysis to determine which locals require allocas and
 //! which do not.
 
+use rustc_abi as abi;
 use rustc_data_structures::graph::dominators::Dominators;
 use rustc_index::bit_set::DenseBitSet;
 use rustc_index::{IndexSlice, IndexVec};
 use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor};
 use rustc_middle::mir::{self, DefLocation, Location, TerminatorKind, traversal};
-use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
+use rustc_middle::ty::layout::LayoutOf;
 use rustc_middle::{bug, span_bug};
 use tracing::debug;
 
@@ -99,63 +100,75 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> LocalAnalyzer<'a, 'b, 'tcx, Bx>
         context: PlaceContext,
         location: Location,
     ) {
-        let cx = self.fx.cx;
-
-        if let Some((place_base, elem)) = place_ref.last_projection() {
-            let mut base_context = if context.is_mutating_use() {
-                PlaceContext::MutatingUse(MutatingUseContext::Projection)
-            } else {
-                PlaceContext::NonMutatingUse(NonMutatingUseContext::Projection)
-            };
-
-            // Allow uses of projections that are ZSTs or from scalar fields.
-            let is_consume = matches!(
-                context,
-                PlaceContext::NonMutatingUse(
-                    NonMutatingUseContext::Copy | NonMutatingUseContext::Move,
-                )
-            );
-            if is_consume {
-                let base_ty = place_base.ty(self.fx.mir, cx.tcx());
-                let base_ty = self.fx.monomorphize(base_ty);
-
-                // ZSTs don't require any actual memory access.
-                let elem_ty = base_ty.projection_ty(cx.tcx(), self.fx.monomorphize(elem)).ty;
-                let span = self.fx.mir.local_decls[place_ref.local].source_info.span;
-                if cx.spanned_layout_of(elem_ty, span).is_zst() {
-                    return;
+        if !place_ref.projection.is_empty() {
+            const COPY_CONTEXT: PlaceContext =
+                PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy);
+
+            // `PlaceElem::Index` is the only variant that can mention other `Local`s,
+            // so check for those up-front before any potential short-circuits.
+            for elem in place_ref.projection {
+                if let mir::PlaceElem::Index(index_local) = *elem {
+                    self.visit_local(index_local, COPY_CONTEXT, location);
                 }
+            }
 
-                if let mir::ProjectionElem::Field(..) = elem {
-                    let layout = cx.spanned_layout_of(base_ty.ty, span);
-                    if cx.is_backend_immediate(layout) || cx.is_backend_scalar_pair(layout) {
-                        // Recurse with the same context, instead of `Projection`,
-                        // potentially stopping at non-operand projections,
-                        // which would trigger `not_ssa` on locals.
-                        base_context = context;
-                    }
-                }
+            // If our local is already memory, nothing can make it *more* memory
+            // so we don't need to bother checking the projections further.
+            if self.locals[place_ref.local] == LocalKind::Memory {
+                return;
             }
 
-            if let mir::ProjectionElem::Deref = elem {
-                // Deref projections typically only read the pointer.
-                base_context = PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy);
+            if place_ref.is_indirect_first_projection() {
+                // If this starts with a `Deref`, we only need to record a read of the
+                // pointer being dereferenced, as all the subsequent projections are
+                // working on a place which is always supported. (And because we're
+                // looking at codegen MIR, it can only happen as the first projection.)
+                self.visit_local(place_ref.local, COPY_CONTEXT, location);
+                return;
             }
 
-            self.process_place(&place_base, base_context, location);
-            // HACK(eddyb) this emulates the old `visit_projection_elem`, this
-            // entire `visit_place`-like `process_place` method should be rewritten,
-            // now that we have moved to the "slice of projections" representation.
-            if let mir::ProjectionElem::Index(local) = elem {
-                self.visit_local(
-                    local,
-                    PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy),
-                    location,
-                );
+            if context.is_mutating_use() {
+                // If it's a mutating use it doesn't matter what the projections are,
+                // if there are *any* then we need a place to write. (For example,
+                // `_1 = Foo()` works in SSA but `_2.0 = Foo()` does not.)
+                let mut_projection = PlaceContext::MutatingUse(MutatingUseContext::Projection);
+                self.visit_local(place_ref.local, mut_projection, location);
+                return;
             }
-        } else {
-            self.visit_local(place_ref.local, context, location);
+
+            // Scan through to ensure the only projections are those which
+            // `FunctionCx::maybe_codegen_consume_direct` can handle.
+            let base_ty = self.fx.monomorphized_place_ty(mir::PlaceRef::from(place_ref.local));
+            let mut layout = self.fx.cx.layout_of(base_ty);
+            for elem in place_ref.projection {
+                layout = match *elem {
+                    mir::PlaceElem::Field(fidx, ..) => layout.field(self.fx.cx, fidx.as_usize()),
+                    mir::PlaceElem::Downcast(_, vidx)
+                        if let abi::Variants::Single { index: single_variant } =
+                            layout.variants
+                            && vidx == single_variant =>
+                    {
+                        layout.for_variant(self.fx.cx, vidx)
+                    }
+                    mir::PlaceElem::Subtype(subtype_ty) => {
+                        let subtype_ty = self.fx.monomorphize(subtype_ty);
+                        self.fx.cx.layout_of(subtype_ty)
+                    }
+                    _ => {
+                        self.locals[place_ref.local] = LocalKind::Memory;
+                        return;
+                    }
+                }
+            }
+            debug_assert!(
+                !self.fx.cx.is_backend_ref(layout),
+                "Post-projection {place_ref:?} layout should be non-Ref, but it's {layout:?}",
+            );
         }
+
+        // Even with supported projections, we still need to have `visit_local`
+        // check for things that can't be done in SSA (like `SharedBorrow`).
+        self.visit_local(place_ref.local, context, location);
     }
 }
 
@@ -170,11 +183,6 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> Visitor<'tcx> for LocalAnalyzer
 
         if let Some(local) = place.as_local() {
             self.define(local, DefLocation::Assignment(location));
-            if self.locals[local] != LocalKind::Memory {
-                if !self.fx.rvalue_creates_operand(rvalue) {
-                    self.locals[local] = LocalKind::Memory;
-                }
-            }
         } else {
             self.visit_place(place, PlaceContext::MutatingUse(MutatingUseContext::Store), location);
         }
diff --git a/compiler/rustc_codegen_ssa/src/mir/constant.rs b/compiler/rustc_codegen_ssa/src/mir/constant.rs
index 68a56044a07..11b6ab3cdf1 100644
--- a/compiler/rustc_codegen_ssa/src/mir/constant.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/constant.rs
@@ -20,7 +20,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         OperandRef::from_const(bx, val, ty)
     }
 
-    pub fn eval_mir_constant(&self, constant: &mir::ConstOperand<'tcx>) -> mir::ConstValue<'tcx> {
+    pub fn eval_mir_constant(&self, constant: &mir::ConstOperand<'tcx>) -> mir::ConstValue {
         // `MirUsedCollector` visited all required_consts before codegen began, so if we got here
         // there can be no more constants that fail to evaluate.
         self.monomorphize(constant.const_)
diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs
index 06bedaaa4a2..5459f95c186 100644
--- a/compiler/rustc_codegen_ssa/src/mir/operand.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs
@@ -140,7 +140,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 
     pub(crate) fn from_const<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
         bx: &mut Bx,
-        val: mir::ConstValue<'tcx>,
+        val: mir::ConstValue,
         ty: Ty<'tcx>,
     ) -> Self {
         let layout = bx.layout_of(ty);
@@ -154,14 +154,11 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
                 OperandValue::Immediate(llval)
             }
             ConstValue::ZeroSized => return OperandRef::zero_sized(layout),
-            ConstValue::Slice { data, meta } => {
+            ConstValue::Slice { alloc_id, meta } => {
                 let BackendRepr::ScalarPair(a_scalar, _) = layout.backend_repr else {
                     bug!("from_const: invalid ScalarPair layout: {:#?}", layout);
                 };
-                let a = Scalar::from_pointer(
-                    Pointer::new(bx.tcx().reserve_and_set_memory_alloc(data).into(), Size::ZERO),
-                    &bx.tcx(),
-                );
+                let a = Scalar::from_pointer(Pointer::new(alloc_id.into(), Size::ZERO), &bx.tcx());
                 let a_llval = bx.scalar_to_backend(
                     a,
                     a_scalar,
@@ -338,13 +335,6 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 
         let val = if field.is_zst() {
             OperandValue::ZeroSized
-        } else if let BackendRepr::SimdVector { .. } = self.layout.backend_repr {
-            // codegen_transmute_operand doesn't support SIMD, but since the previous
-            // check handled ZSTs, the only possible field access into something SIMD
-            // is to the `non_1zst_field` that's the same SIMD. (Other things, even
-            // just padding, would change the wrapper's representation type.)
-            assert_eq!(field.size, self.layout.size);
-            self.val
         } else if field.size == self.layout.size {
             assert_eq!(offset.bytes(), 0);
             fx.codegen_transmute_operand(bx, *self, field)
@@ -931,9 +921,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
         match self.locals[place_ref.local] {
             LocalRef::Operand(mut o) => {
-                // Moves out of scalar and scalar pair fields are trivial.
-                for elem in place_ref.projection.iter() {
-                    match elem {
+                // We only need to handle the projections that
+                // `LocalAnalyzer::process_place` let make it here.
+                for elem in place_ref.projection {
+                    match *elem {
                         mir::ProjectionElem::Field(f, _) => {
                             assert!(
                                 !o.layout.ty.is_any_ptr(),
@@ -942,17 +933,18 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             );
                             o = o.extract_field(self, bx, f.index());
                         }
-                        mir::ProjectionElem::Index(_)
-                        | mir::ProjectionElem::ConstantIndex { .. } => {
-                            // ZSTs don't require any actual memory access.
-                            // FIXME(eddyb) deduplicate this with the identical
-                            // checks in `codegen_consume` and `extract_field`.
-                            let elem = o.layout.field(bx.cx(), 0);
-                            if elem.is_zst() {
-                                o = OperandRef::zero_sized(elem);
-                            } else {
-                                return None;
-                            }
+                        mir::PlaceElem::Downcast(_, vidx) => {
+                            debug_assert_eq!(
+                                o.layout.variants,
+                                abi::Variants::Single { index: vidx },
+                            );
+                            let layout = o.layout.for_variant(bx.cx(), vidx);
+                            o = OperandRef { val: o.val, layout }
+                        }
+                        mir::PlaceElem::Subtype(subtype_ty) => {
+                            let subtype_ty = self.monomorphize(subtype_ty);
+                            let layout = self.cx.layout_of(subtype_ty);
+                            o = OperandRef { val: o.val, layout }
                         }
                         _ => return None,
                     }
diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
index 610e2fd2311..8a67b8d6e5f 100644
--- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
@@ -2,12 +2,12 @@ use rustc_abi::{self as abi, FIRST_VARIANT};
 use rustc_middle::ty::adjustment::PointerCoercion;
 use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, LayoutOf, TyAndLayout};
 use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
-use rustc_middle::{bug, mir};
+use rustc_middle::{bug, mir, span_bug};
 use rustc_session::config::OptLevel;
 use tracing::{debug, instrument};
 
 use super::operand::{OperandRef, OperandRefBuilder, OperandValue};
-use super::place::{PlaceRef, codegen_tag_value};
+use super::place::{PlaceRef, PlaceValue, codegen_tag_value};
 use super::{FunctionCx, LocalRef};
 use crate::common::{IntPredicate, TypeKind};
 use crate::traits::*;
@@ -180,7 +180,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             }
 
             _ => {
-                assert!(self.rvalue_creates_operand(rvalue));
                 let temp = self.codegen_rvalue_operand(bx, rvalue);
                 temp.val.store(bx, dest);
             }
@@ -218,17 +217,26 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
     /// Transmutes an `OperandValue` to another `OperandValue`.
     ///
-    /// This is supported only for cases where [`Self::rvalue_creates_operand`]
-    /// returns `true`, and will ICE otherwise. (In particular, anything that
-    /// would need to `alloca` in order to return a `PlaceValue` will ICE,
-    /// expecting those to go via [`Self::codegen_transmute`] instead where
-    /// the destination place is already allocated.)
+    /// This is supported for all cases where the `cast` type is SSA,
+    /// but for non-ZSTs with [`abi::BackendRepr::Memory`] it ICEs.
     pub(crate) fn codegen_transmute_operand(
         &mut self,
         bx: &mut Bx,
         operand: OperandRef<'tcx, Bx::Value>,
         cast: TyAndLayout<'tcx>,
     ) -> OperandValue<Bx::Value> {
+        if let abi::BackendRepr::Memory { .. } = cast.backend_repr
+            && !cast.is_zst()
+        {
+            span_bug!(self.mir.span, "Use `codegen_transmute` to transmute to {cast:?}");
+        }
+
+        // `Layout` is interned, so we can do a cheap check for things that are
+        // exactly the same and thus don't need any handling.
+        if abi::Layout::eq(&operand.layout.layout, &cast.layout) {
+            return operand.val;
+        }
+
         // Check for transmutes that are always UB.
         if operand.layout.size != cast.size
             || operand.layout.is_uninhabited()
@@ -241,11 +249,22 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             return OperandValue::poison(bx, cast);
         }
 
+        // To or from pointers takes different methods, so we use this to restrict
+        // the SimdVector case to types which can be `bitcast` between each other.
+        #[inline]
+        fn vector_can_bitcast(x: abi::Scalar) -> bool {
+            matches!(
+                x,
+                abi::Scalar::Initialized {
+                    value: abi::Primitive::Int(..) | abi::Primitive::Float(..),
+                    ..
+                }
+            )
+        }
+
+        let cx = bx.cx();
         match (operand.val, operand.layout.backend_repr, cast.backend_repr) {
             _ if cast.is_zst() => OperandValue::ZeroSized,
-            (_, _, abi::BackendRepr::Memory { .. }) => {
-                bug!("Cannot `codegen_transmute_operand` to non-ZST memory-ABI output {cast:?}");
-            }
             (OperandValue::Ref(source_place_val), abi::BackendRepr::Memory { .. }, _) => {
                 assert_eq!(source_place_val.llextra, None);
                 // The existing alignment is part of `source_place_val`,
@@ -256,16 +275,46 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 OperandValue::Immediate(imm),
                 abi::BackendRepr::Scalar(from_scalar),
                 abi::BackendRepr::Scalar(to_scalar),
-            ) => OperandValue::Immediate(transmute_scalar(bx, imm, from_scalar, to_scalar)),
+            ) if from_scalar.size(cx) == to_scalar.size(cx) => {
+                OperandValue::Immediate(transmute_scalar(bx, imm, from_scalar, to_scalar))
+            }
+            (
+                OperandValue::Immediate(imm),
+                abi::BackendRepr::SimdVector { element: from_scalar, .. },
+                abi::BackendRepr::SimdVector { element: to_scalar, .. },
+            ) if vector_can_bitcast(from_scalar) && vector_can_bitcast(to_scalar) => {
+                let to_backend_ty = bx.cx().immediate_backend_type(cast);
+                OperandValue::Immediate(bx.bitcast(imm, to_backend_ty))
+            }
             (
                 OperandValue::Pair(imm_a, imm_b),
                 abi::BackendRepr::ScalarPair(in_a, in_b),
                 abi::BackendRepr::ScalarPair(out_a, out_b),
-            ) => OperandValue::Pair(
-                transmute_scalar(bx, imm_a, in_a, out_a),
-                transmute_scalar(bx, imm_b, in_b, out_b),
-            ),
-            _ => bug!("Cannot `codegen_transmute_operand` {operand:?} to {cast:?}"),
+            ) if in_a.size(cx) == out_a.size(cx) && in_b.size(cx) == out_b.size(cx) => {
+                OperandValue::Pair(
+                    transmute_scalar(bx, imm_a, in_a, out_a),
+                    transmute_scalar(bx, imm_b, in_b, out_b),
+                )
+            }
+            _ => {
+                // For any other potentially-tricky cases, make a temporary instead.
+                // If anything else wants the target local to be in memory this won't
+                // be hit, as `codegen_transmute` will get called directly. Thus this
+                // is only for places where everything else wants the operand form,
+                // and thus it's not worth making those places get it from memory.
+                //
+                // Notably, Scalar ⇌ ScalarPair cases go here to avoid padding
+                // and endianness issues, as do SimdVector ones to avoid worrying
+                // about things like f32x8 ⇌ ptrx4 that would need multiple steps.
+                let align = Ord::max(operand.layout.align.abi, cast.align.abi);
+                let size = Ord::max(operand.layout.size, cast.size);
+                let temp = PlaceValue::alloca(bx, size, align);
+                bx.lifetime_start(temp.llval, size);
+                operand.val.store(bx, temp.with_type(operand.layout));
+                let val = bx.load_operand(temp.with_type(cast)).val;
+                bx.lifetime_end(temp.llval, size);
+                val
+            }
         }
     }
 
@@ -288,7 +337,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         // valid ranges. For example, `char`s are passed as just `i32`, with no
         // way for LLVM to know that they're 0x10FFFF at most. Thus we assume
         // the range of the input value too, not just the output range.
-        assume_scalar_range(bx, imm, from_scalar, from_backend_ty);
+        assume_scalar_range(bx, imm, from_scalar, from_backend_ty, None);
 
         imm = match (from_scalar.primitive(), to_scalar.primitive()) {
             (Int(_, is_signed), Int(..)) => bx.intcast(imm, to_backend_ty, is_signed),
@@ -326,8 +375,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         bx: &mut Bx,
         rvalue: &mir::Rvalue<'tcx>,
     ) -> OperandRef<'tcx, Bx::Value> {
-        assert!(self.rvalue_creates_operand(rvalue), "cannot codegen {rvalue:?} to operand",);
-
         match *rvalue {
             mir::Rvalue::Cast(ref kind, ref source, mir_cast_ty) => {
                 let operand = self.codegen_operand(bx, source);
@@ -653,8 +700,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 let ty = self.monomorphize(ty);
                 let layout = self.cx.layout_of(ty);
 
-                // `rvalue_creates_operand` has arranged that we only get here if
-                // we can build the aggregate immediate from the field immediates.
                 let mut builder = OperandRefBuilder::new(layout);
                 for (field_idx, field) in fields.iter_enumerated() {
                     let op = self.codegen_operand(bx, field);
@@ -955,69 +1000,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
         OperandValue::Pair(val, of)
     }
-
-    /// Returns `true` if the `rvalue` can be computed into an [`OperandRef`],
-    /// rather than needing a full `PlaceRef` for the assignment destination.
-    ///
-    /// This is used by the [`super::analyze`] code to decide which MIR locals
-    /// can stay as SSA values (as opposed to generating `alloca` slots for them).
-    /// As such, some paths here return `true` even where the specific rvalue
-    /// will not actually take the operand path because the result type is such
-    /// that it always gets an `alloca`, but where it's not worth re-checking the
-    /// layout in this code when the right thing will happen anyway.
-    pub(crate) fn rvalue_creates_operand(&self, rvalue: &mir::Rvalue<'tcx>) -> bool {
-        match *rvalue {
-            mir::Rvalue::Cast(mir::CastKind::Transmute, ref operand, cast_ty) => {
-                let operand_ty = operand.ty(self.mir, self.cx.tcx());
-                let cast_layout = self.cx.layout_of(self.monomorphize(cast_ty));
-                let operand_layout = self.cx.layout_of(self.monomorphize(operand_ty));
-                match (operand_layout.backend_repr, cast_layout.backend_repr) {
-                    // When the output will be in memory anyway, just use its place
-                    // (instead of the operand path) unless it's the trivial ZST case.
-                    (_, abi::BackendRepr::Memory { .. }) => cast_layout.is_zst(),
-
-                    // Otherwise (for a non-memory output) if the input is memory
-                    // then we can just read the value from the place.
-                    (abi::BackendRepr::Memory { .. }, _) => true,
-
-                    // When we have scalar immediates, we can only convert things
-                    // where the sizes match, to avoid endianness questions.
-                    (abi::BackendRepr::Scalar(a), abi::BackendRepr::Scalar(b)) =>
-                        a.size(self.cx) == b.size(self.cx),
-                    (abi::BackendRepr::ScalarPair(a0, a1), abi::BackendRepr::ScalarPair(b0, b1)) =>
-                        a0.size(self.cx) == b0.size(self.cx) && a1.size(self.cx) == b1.size(self.cx),
-
-                    // Mixing Scalars and ScalarPairs can get quite complicated when
-                    // padding and undef get involved, so leave that to the memory path.
-                    (abi::BackendRepr::Scalar(_), abi::BackendRepr::ScalarPair(_, _)) |
-                    (abi::BackendRepr::ScalarPair(_, _), abi::BackendRepr::Scalar(_)) => false,
-
-                    // SIMD vectors aren't worth the trouble of dealing with complex
-                    // cases like from vectors of f32 to vectors of pointers or
-                    // from fat pointers to vectors of u16. (See #143194 #110021 ...)
-                    (abi::BackendRepr::SimdVector { .. }, _) | (_, abi::BackendRepr::SimdVector { .. }) => false,
-                }
-            }
-            mir::Rvalue::Ref(..) |
-            mir::Rvalue::CopyForDeref(..) |
-            mir::Rvalue::RawPtr(..) |
-            mir::Rvalue::Len(..) |
-            mir::Rvalue::Cast(..) | // (*)
-            mir::Rvalue::ShallowInitBox(..) | // (*)
-            mir::Rvalue::BinaryOp(..) |
-            mir::Rvalue::UnaryOp(..) |
-            mir::Rvalue::Discriminant(..) |
-            mir::Rvalue::NullaryOp(..) |
-            mir::Rvalue::ThreadLocalRef(_) |
-            mir::Rvalue::Use(..) |
-            mir::Rvalue::Repeat(..) | // (*)
-            mir::Rvalue::Aggregate(..) | // (*)
-            mir::Rvalue::WrapUnsafeBinder(..) => // (*)
-                true,
-        }
-
-        // (*) this is only true if the type is suitable
-    }
 }
 
 /// Transmutes a single scalar value `imm` from `from_scalar` to `to_scalar`.
@@ -1064,7 +1046,7 @@ pub(super) fn transmute_scalar<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     // That said, last time we tried removing this, it didn't actually help
     // the rustc-perf results, so might as well keep doing it
     // <https://github.com/rust-lang/rust/pull/135610#issuecomment-2599275182>
-    assume_scalar_range(bx, imm, from_scalar, from_backend_ty);
+    assume_scalar_range(bx, imm, from_scalar, from_backend_ty, Some(&to_scalar));
 
     imm = match (from_scalar.primitive(), to_scalar.primitive()) {
         (Int(..) | Float(_), Int(..) | Float(_)) => bx.bitcast(imm, to_backend_ty),
@@ -1092,22 +1074,42 @@ pub(super) fn transmute_scalar<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     // since it's never passed to something with parameter metadata (especially
     // after MIR inlining) so the only way to tell the backend about the
     // constraint that the `transmute` introduced is to `assume` it.
-    assume_scalar_range(bx, imm, to_scalar, to_backend_ty);
+    assume_scalar_range(bx, imm, to_scalar, to_backend_ty, Some(&from_scalar));
 
     imm = bx.to_immediate_scalar(imm, to_scalar);
     imm
 }
 
+/// Emits an `assume` call that `imm`'s value is within the known range of `scalar`.
+///
+/// If `known` is `Some`, only emits the assume if it's more specific than
+/// whatever is already known from the range of *that* scalar.
 fn assume_scalar_range<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     bx: &mut Bx,
     imm: Bx::Value,
     scalar: abi::Scalar,
     backend_ty: Bx::Type,
+    known: Option<&abi::Scalar>,
 ) {
-    if matches!(bx.cx().sess().opts.optimize, OptLevel::No) || scalar.is_always_valid(bx.cx()) {
+    if matches!(bx.cx().sess().opts.optimize, OptLevel::No) {
         return;
     }
 
+    match (scalar, known) {
+        (abi::Scalar::Union { .. }, _) => return,
+        (_, None) => {
+            if scalar.is_always_valid(bx.cx()) {
+                return;
+            }
+        }
+        (abi::Scalar::Initialized { valid_range, .. }, Some(known)) => {
+            let known_range = known.valid_range(bx.cx());
+            if valid_range.contains_range(known_range, scalar.size(bx.cx())) {
+                return;
+            }
+        }
+    }
+
     match scalar.primitive() {
         abi::Primitive::Int(..) => {
             let range = scalar.valid_range(bx.cx());
diff --git a/compiler/rustc_codegen_ssa/src/traits/write.rs b/compiler/rustc_codegen_ssa/src/traits/write.rs
index 5e993640472..8e78cbe1963 100644
--- a/compiler/rustc_codegen_ssa/src/traits/write.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/write.rs
@@ -1,3 +1,5 @@
+use std::path::PathBuf;
+
 use rustc_ast::expand::autodiff_attrs::AutoDiffItem;
 use rustc_errors::{DiagCtxtHandle, FatalError};
 use rustc_middle::dep_graph::WorkProduct;
@@ -24,8 +26,9 @@ pub trait WriteBackendMethods: Clone + 'static {
     /// if necessary and running any further optimizations
     fn run_and_optimize_fat_lto(
         cgcx: &CodegenContext<Self>,
+        exported_symbols_for_lto: &[String],
+        each_linked_rlib_for_lto: &[PathBuf],
         modules: Vec<FatLtoInput<Self>>,
-        cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>,
         diff_fncs: Vec<AutoDiffItem>,
     ) -> Result<ModuleCodegen<Self::Module>, FatalError>;
     /// Performs thin LTO by performing necessary global analysis and returning two
@@ -33,6 +36,8 @@ pub trait WriteBackendMethods: Clone + 'static {
     /// can simply be copied over from the incr. comp. cache.
     fn run_thin_lto(
         cgcx: &CodegenContext<Self>,
+        exported_symbols_for_lto: &[String],
+        each_linked_rlib_for_lto: &[PathBuf],
         modules: Vec<(String, Self::ThinBuffer)>,
         cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>,
     ) -> Result<(Vec<ThinModule<Self>>, Vec<WorkProduct>), FatalError>;
diff --git a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
index f584f6c948e..5835660e1c3 100644
--- a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
+++ b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
@@ -152,7 +152,7 @@ pub(crate) fn mk_eval_cx_to_read_const_val<'tcx>(
 pub fn mk_eval_cx_for_const_val<'tcx>(
     tcx: TyCtxtAt<'tcx>,
     typing_env: ty::TypingEnv<'tcx>,
-    val: mir::ConstValue<'tcx>,
+    val: mir::ConstValue,
     ty: Ty<'tcx>,
 ) -> Option<(CompileTimeInterpCx<'tcx>, OpTy<'tcx>)> {
     let ecx = mk_eval_cx_to_read_const_val(tcx.tcx, tcx.span, typing_env, CanAccessMutGlobal::No);
@@ -172,7 +172,7 @@ pub(super) fn op_to_const<'tcx>(
     ecx: &CompileTimeInterpCx<'tcx>,
     op: &OpTy<'tcx>,
     for_diagnostics: bool,
-) -> ConstValue<'tcx> {
+) -> ConstValue {
     // Handle ZST consistently and early.
     if op.layout.is_zst() {
         return ConstValue::ZeroSized;
@@ -241,10 +241,9 @@ pub(super) fn op_to_const<'tcx>(
                 let (prov, offset) =
                     ptr.into_pointer_or_addr().expect(msg).prov_and_relative_offset();
                 let alloc_id = prov.alloc_id();
-                let data = ecx.tcx.global_alloc(alloc_id).unwrap_memory();
                 assert!(offset == abi::Size::ZERO, "{}", msg);
                 let meta = b.to_target_usize(ecx).expect(msg);
-                ConstValue::Slice { data, meta }
+                ConstValue::Slice { alloc_id, meta }
             }
             Immediate::Uninit => bug!("`Uninit` is not a valid value for {}", op.layout.ty),
         },
@@ -256,7 +255,7 @@ pub(crate) fn turn_into_const_value<'tcx>(
     tcx: TyCtxt<'tcx>,
     constant: ConstAlloc<'tcx>,
     key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>,
-) -> ConstValue<'tcx> {
+) -> ConstValue {
     let cid = key.value;
     let def_id = cid.instance.def.def_id();
     let is_static = tcx.is_static(def_id);
diff --git a/compiler/rustc_const_eval/src/const_eval/mod.rs b/compiler/rustc_const_eval/src/const_eval/mod.rs
index 0082f90f3b8..624ca1dd2da 100644
--- a/compiler/rustc_const_eval/src/const_eval/mod.rs
+++ b/compiler/rustc_const_eval/src/const_eval/mod.rs
@@ -28,7 +28,7 @@ const VALTREE_MAX_NODES: usize = 100000;
 #[instrument(skip(tcx), level = "debug")]
 pub(crate) fn try_destructure_mir_constant_for_user_output<'tcx>(
     tcx: TyCtxt<'tcx>,
-    val: mir::ConstValue<'tcx>,
+    val: mir::ConstValue,
     ty: Ty<'tcx>,
 ) -> Option<mir::DestructuredConstant<'tcx>> {
     let typing_env = ty::TypingEnv::fully_monomorphized();
diff --git a/compiler/rustc_const_eval/src/const_eval/valtrees.rs b/compiler/rustc_const_eval/src/const_eval/valtrees.rs
index 5ab72c853c4..37c6c4a61d8 100644
--- a/compiler/rustc_const_eval/src/const_eval/valtrees.rs
+++ b/compiler/rustc_const_eval/src/const_eval/valtrees.rs
@@ -259,7 +259,7 @@ pub fn valtree_to_const_value<'tcx>(
     tcx: TyCtxt<'tcx>,
     typing_env: ty::TypingEnv<'tcx>,
     cv: ty::Value<'tcx>,
-) -> mir::ConstValue<'tcx> {
+) -> mir::ConstValue {
     // Basic idea: We directly construct `Scalar` values from trivial `ValTree`s
     // (those for constants with type bool, int, uint, float or char).
     // For all other types we create an `MPlace` and fill that by walking
diff --git a/compiler/rustc_const_eval/src/interpret/eval_context.rs b/compiler/rustc_const_eval/src/interpret/eval_context.rs
index 11e7706fe60..0c888694e49 100644
--- a/compiler/rustc_const_eval/src/interpret/eval_context.rs
+++ b/compiler/rustc_const_eval/src/interpret/eval_context.rs
@@ -582,8 +582,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         span: Span,
         layout: Option<TyAndLayout<'tcx>>,
     ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
-        M::eval_mir_constant(self, *val, span, layout, |ecx, val, span, layout| {
-            let const_val = val.eval(*ecx.tcx, ecx.typing_env, span).map_err(|err| {
+        let const_val = val.eval(*self.tcx, self.typing_env, span).map_err(|err| {
                 if M::ALL_CONSTS_ARE_PRECHECKED {
                     match err {
                         ErrorHandled::TooGeneric(..) => {},
@@ -599,11 +598,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                         }
                     }
                 }
-                err.emit_note(*ecx.tcx);
+                err.emit_note(*self.tcx);
                 err
             })?;
-            ecx.const_val_to_op(const_val, val.ty(), layout)
-        })
+        self.const_val_to_op(const_val, val.ty(), layout)
     }
 
     #[must_use]
diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
index e24a355891d..5e3d0a15d8b 100644
--- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs
+++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
@@ -6,7 +6,7 @@ use std::assert_matches::assert_matches;
 
 use rustc_abi::{FieldIdx, HasDataLayout, Size};
 use rustc_apfloat::ieee::{Double, Half, Quad, Single};
-use rustc_middle::mir::interpret::{read_target_uint, write_target_uint};
+use rustc_middle::mir::interpret::{CTFE_ALLOC_SALT, read_target_uint, write_target_uint};
 use rustc_middle::mir::{self, BinOp, ConstValue, NonDivergingIntrinsic};
 use rustc_middle::ty::layout::TyAndLayout;
 use rustc_middle::ty::{Ty, TyCtxt};
@@ -17,17 +17,18 @@ use tracing::trace;
 use super::memory::MemoryKind;
 use super::util::ensure_monomorphic_enough;
 use super::{
-    Allocation, CheckInAllocMsg, ConstAllocation, ImmTy, InterpCx, InterpResult, Machine, OpTy,
-    PlaceTy, Pointer, PointerArithmetic, Provenance, Scalar, err_ub_custom, err_unsup_format,
-    interp_ok, throw_inval, throw_ub_custom, throw_ub_format,
+    AllocId, CheckInAllocMsg, ImmTy, InterpCx, InterpResult, Machine, OpTy, PlaceTy, Pointer,
+    PointerArithmetic, Provenance, Scalar, err_ub_custom, err_unsup_format, interp_ok, throw_inval,
+    throw_ub_custom, throw_ub_format,
 };
 use crate::fluent_generated as fluent;
 
 /// Directly returns an `Allocation` containing an absolute path representation of the given type.
-pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ConstAllocation<'tcx> {
+pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> (AllocId, u64) {
     let path = crate::util::type_name(tcx, ty);
-    let alloc = Allocation::from_bytes_byte_aligned_immutable(path.into_bytes(), ());
-    tcx.mk_const_alloc(alloc)
+    let bytes = path.into_bytes();
+    let len = bytes.len().try_into().unwrap();
+    (tcx.allocate_bytes_dedup(bytes, CTFE_ALLOC_SALT), len)
 }
 impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     /// Generates a value of `TypeId` for `ty` in-place.
@@ -126,8 +127,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             sym::type_name => {
                 let tp_ty = instance.args.type_at(0);
                 ensure_monomorphic_enough(tcx, tp_ty)?;
-                let alloc = alloc_type_name(tcx, tp_ty);
-                let val = ConstValue::Slice { data: alloc, meta: alloc.inner().size().bytes() };
+                let (alloc_id, meta) = alloc_type_name(tcx, tp_ty);
+                let val = ConstValue::Slice { alloc_id, meta };
                 let val = self.const_val_to_op(val, dest.layout.ty, Some(dest.layout))?;
                 self.copy_op(&val, dest)?;
             }
diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs
index e981f3973ae..e22629993fb 100644
--- a/compiler/rustc_const_eval/src/interpret/machine.rs
+++ b/compiler/rustc_const_eval/src/interpret/machine.rs
@@ -12,7 +12,6 @@ use rustc_middle::query::TyCtxtAt;
 use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::TyAndLayout;
 use rustc_middle::{mir, ty};
-use rustc_span::Span;
 use rustc_span::def_id::DefId;
 use rustc_target::callconv::FnAbi;
 
@@ -587,27 +586,6 @@ pub trait Machine<'tcx>: Sized {
         interp_ok(())
     }
 
-    /// Evaluate the given constant. The `eval` function will do all the required evaluation,
-    /// but this hook has the chance to do some pre/postprocessing.
-    #[inline(always)]
-    fn eval_mir_constant<F>(
-        ecx: &InterpCx<'tcx, Self>,
-        val: mir::Const<'tcx>,
-        span: Span,
-        layout: Option<TyAndLayout<'tcx>>,
-        eval: F,
-    ) -> InterpResult<'tcx, OpTy<'tcx, Self::Provenance>>
-    where
-        F: Fn(
-            &InterpCx<'tcx, Self>,
-            mir::Const<'tcx>,
-            Span,
-            Option<TyAndLayout<'tcx>>,
-        ) -> InterpResult<'tcx, OpTy<'tcx, Self::Provenance>>,
-    {
-        eval(ecx, val, span, layout)
-    }
-
     /// Returns the salt to be used for a deduplicated global alloation.
     /// If the allocation is for a function, the instance is provided as well
     /// (this lets Miri ensure unique addresses for some functions).
diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs
index 62cbbae24a8..21afd082a05 100644
--- a/compiler/rustc_const_eval/src/interpret/operand.rs
+++ b/compiler/rustc_const_eval/src/interpret/operand.rs
@@ -836,7 +836,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
 
     pub(crate) fn const_val_to_op(
         &self,
-        val_val: mir::ConstValue<'tcx>,
+        val_val: mir::ConstValue,
         ty: Ty<'tcx>,
         layout: Option<TyAndLayout<'tcx>>,
     ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
@@ -860,9 +860,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             }
             mir::ConstValue::Scalar(x) => adjust_scalar(x)?.into(),
             mir::ConstValue::ZeroSized => Immediate::Uninit,
-            mir::ConstValue::Slice { data, meta } => {
+            mir::ConstValue::Slice { alloc_id, meta } => {
                 // This is const data, no mutation allowed.
-                let alloc_id = self.tcx.reserve_and_set_memory_alloc(data);
                 let ptr = Pointer::new(CtfeProvenance::from(alloc_id).as_immutable(), Size::ZERO);
                 Immediate::new_slice(self.global_root_pointer(ptr)?.into(), meta, self)
             }
diff --git a/compiler/rustc_const_eval/src/util/caller_location.rs b/compiler/rustc_const_eval/src/util/caller_location.rs
index f489b05fbbd..c437934eaab 100644
--- a/compiler/rustc_const_eval/src/util/caller_location.rs
+++ b/compiler/rustc_const_eval/src/util/caller_location.rs
@@ -57,7 +57,7 @@ pub(crate) fn const_caller_location_provider(
     file: Symbol,
     line: u32,
     col: u32,
-) -> mir::ConstValue<'_> {
+) -> mir::ConstValue {
     trace!("const_caller_location: {}:{}:{}", file, line, col);
     let mut ecx = mk_eval_cx_to_read_const_val(
         tcx,
diff --git a/compiler/rustc_data_structures/src/profiling.rs b/compiler/rustc_data_structures/src/profiling.rs
index 881aa679156..4a9551a60cf 100644
--- a/compiler/rustc_data_structures/src/profiling.rs
+++ b/compiler/rustc_data_structures/src/profiling.rs
@@ -551,6 +551,11 @@ impl SelfProfilerRef {
     pub fn get_self_profiler(&self) -> Option<Arc<SelfProfiler>> {
         self.profiler.clone()
     }
+
+    /// Is expensive recording of query keys and/or function arguments enabled?
+    pub fn is_args_recording_enabled(&self) -> bool {
+        self.enabled() && self.event_filter_mask.intersects(EventFilter::ARGS)
+    }
 }
 
 /// A helper for recording costly arguments to self-profiling events. Used with
diff --git a/compiler/rustc_errors/src/markdown/term.rs b/compiler/rustc_errors/src/markdown/term.rs
index 579e00b8b85..fe1d80bdbe8 100644
--- a/compiler/rustc_errors/src/markdown/term.rs
+++ b/compiler/rustc_errors/src/markdown/term.rs
@@ -18,7 +18,7 @@ thread_local! {
 pub(crate) fn entrypoint(stream: &MdStream<'_>, buf: &mut Buffer) -> io::Result<()> {
     #[cfg(not(test))]
     if let Some((w, _)) = termize::dimensions() {
-        WIDTH.with(|c| c.set(std::cmp::min(w, DEFAULT_COLUMN_WIDTH)));
+        WIDTH.set(std::cmp::min(w, DEFAULT_COLUMN_WIDTH));
     }
     write_stream(stream, buf, None, 0)?;
     buf.write_all(b"\n")
@@ -84,7 +84,7 @@ fn write_tt(tt: &MdTree<'_>, buf: &mut Buffer, indent: usize) -> io::Result<()>
             reset_cursor();
         }
         MdTree::HorizontalRule => {
-            (0..WIDTH.with(Cell::get)).for_each(|_| buf.write_all(b"-").unwrap());
+            (0..WIDTH.get()).for_each(|_| buf.write_all(b"-").unwrap());
             reset_cursor();
         }
         MdTree::Heading(n, stream) => {
@@ -121,7 +121,7 @@ fn write_tt(tt: &MdTree<'_>, buf: &mut Buffer, indent: usize) -> io::Result<()>
 
 /// End of that block, just wrap the line
 fn reset_cursor() {
-    CURSOR.with(|cur| cur.set(0));
+    CURSOR.set(0);
 }
 
 /// Change to be generic on Write for testing. If we have a link URL, we don't
@@ -144,7 +144,7 @@ fn write_wrapping<B: io::Write>(
                 buf.write_all(ind_ws)?;
                 cur.set(indent);
             }
-            let ch_count = WIDTH.with(Cell::get) - cur.get();
+            let ch_count = WIDTH.get() - cur.get();
             let mut iter = to_write.char_indices();
             let Some((end_idx, _ch)) = iter.nth(ch_count) else {
                 // Write entire line
diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs
index 25ec5401111..5a7883ccba6 100644
--- a/compiler/rustc_expand/src/base.rs
+++ b/compiler/rustc_expand/src/base.rs
@@ -1141,7 +1141,7 @@ pub trait ResolverExpand {
 
     /// Names of specific methods to which glob delegation expands.
     fn glob_delegation_suffixes(
-        &mut self,
+        &self,
         trait_def_id: DefId,
         impl_def_id: LocalDefId,
     ) -> Result<Vec<(Ident, Option<Ident>)>, Indeterminate>;
@@ -1224,6 +1224,7 @@ pub struct ExtCtxt<'a> {
     pub(super) expanded_inert_attrs: MarkedAttrs,
     /// `-Zmacro-stats` data.
     pub macro_stats: FxHashMap<(Symbol, MacroKind), MacroStat>,
+    pub nb_macro_errors: usize,
 }
 
 impl<'a> ExtCtxt<'a> {
@@ -1254,6 +1255,7 @@ impl<'a> ExtCtxt<'a> {
             expanded_inert_attrs: MarkedAttrs::new(),
             buffered_early_lint: vec![],
             macro_stats: Default::default(),
+            nb_macro_errors: 0,
         }
     }
 
@@ -1315,6 +1317,12 @@ impl<'a> ExtCtxt<'a> {
         self.current_expansion.id.expansion_cause()
     }
 
+    /// This method increases the internal macro errors count and then call `trace_macros_diag`.
+    pub fn macro_error_and_trace_macros_diag(&mut self) {
+        self.nb_macro_errors += 1;
+        self.trace_macros_diag();
+    }
+
     pub fn trace_macros_diag(&mut self) {
         for (span, notes) in self.expansions.iter() {
             let mut db = self.dcx().create_note(errors::TraceMacro { span: *span });
diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs
index 79ec79a2fdf..0517fd0419d 100644
--- a/compiler/rustc_expand/src/expand.rs
+++ b/compiler/rustc_expand/src/expand.rs
@@ -693,7 +693,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             crate_name: self.cx.ecfg.crate_name,
         });
 
-        self.cx.trace_macros_diag();
+        self.cx.macro_error_and_trace_macros_diag();
         guar
     }
 
@@ -707,7 +707,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
     ) -> ErrorGuaranteed {
         let guar =
             self.cx.dcx().emit_err(WrongFragmentKind { span, kind: kind.name(), name: &mac.path });
-        self.cx.trace_macros_diag();
+        self.cx.macro_error_and_trace_macros_diag();
         guar
     }
 
@@ -1048,7 +1048,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                 }
                 annotate_err_with_kind(&mut err, kind, span);
                 let guar = err.emit();
-                self.cx.trace_macros_diag();
+                self.cx.macro_error_and_trace_macros_diag();
                 kind.dummy(span, guar)
             }
         }
diff --git a/compiler/rustc_expand/src/mbe/macro_parser.rs b/compiler/rustc_expand/src/mbe/macro_parser.rs
index 3f1fc841ea3..0324057e331 100644
--- a/compiler/rustc_expand/src/mbe/macro_parser.rs
+++ b/compiler/rustc_expand/src/mbe/macro_parser.rs
@@ -299,6 +299,7 @@ enum EofMatcherPositions {
 }
 
 /// Represents the possible results of an attempted parse.
+#[derive(Debug)]
 pub(crate) enum ParseResult<T, F> {
     /// Parsed successfully.
     Success(T),
diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs
index 2f713a09b95..febe6f8b88c 100644
--- a/compiler/rustc_expand/src/mbe/macro_rules.rs
+++ b/compiler/rustc_expand/src/mbe/macro_rules.rs
@@ -280,7 +280,7 @@ fn expand_macro<'cx>(
             // Retry and emit a better error.
             let (span, guar) =
                 diagnostics::failed_to_match_macro(cx.psess(), sp, def_span, name, arg, rules);
-            cx.trace_macros_diag();
+            cx.macro_error_and_trace_macros_diag();
             DummyResult::any(span, guar)
         }
     }
diff --git a/compiler/rustc_hir_analysis/src/check/check.rs b/compiler/rustc_hir_analysis/src/check/check.rs
index 03c026cd6c8..acc831d30ca 100644
--- a/compiler/rustc_hir_analysis/src/check/check.rs
+++ b/compiler/rustc_hir_analysis/src/check/check.rs
@@ -767,7 +767,8 @@ pub(crate) fn check_item_type(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Result<(),
                 DefKind::Static { .. } => {
                     check_static_inhabited(tcx, def_id);
                     check_static_linkage(tcx, def_id);
-                    res = res.and(wfcheck::check_static_item(tcx, def_id));
+                    let ty = tcx.type_of(def_id).instantiate_identity();
+                    res = res.and(wfcheck::check_static_item(tcx, def_id, ty, true));
                 }
                 DefKind::Const => res = res.and(wfcheck::check_const_item(tcx, def_id)),
                 _ => unreachable!(),
@@ -1642,20 +1643,40 @@ fn check_enum(tcx: TyCtxt<'_>, def_id: LocalDefId) {
 
     if def.repr().int.is_none() {
         let is_unit = |var: &ty::VariantDef| matches!(var.ctor_kind(), Some(CtorKind::Const));
-        let has_disr = |var: &ty::VariantDef| matches!(var.discr, ty::VariantDiscr::Explicit(_));
+        let get_disr = |var: &ty::VariantDef| match var.discr {
+            ty::VariantDiscr::Explicit(disr) => Some(disr),
+            ty::VariantDiscr::Relative(_) => None,
+        };
 
-        let has_non_units = def.variants().iter().any(|var| !is_unit(var));
-        let disr_units = def.variants().iter().any(|var| is_unit(var) && has_disr(var));
-        let disr_non_unit = def.variants().iter().any(|var| !is_unit(var) && has_disr(var));
+        let non_unit = def.variants().iter().find(|var| !is_unit(var));
+        let disr_unit =
+            def.variants().iter().filter(|var| is_unit(var)).find_map(|var| get_disr(var));
+        let disr_non_unit =
+            def.variants().iter().filter(|var| !is_unit(var)).find_map(|var| get_disr(var));
 
-        if disr_non_unit || (disr_units && has_non_units) {
-            struct_span_code_err!(
+        if disr_non_unit.is_some() || (disr_unit.is_some() && non_unit.is_some()) {
+            let mut err = struct_span_code_err!(
                 tcx.dcx(),
                 tcx.def_span(def_id),
                 E0732,
-                "`#[repr(inttype)]` must be specified"
-            )
-            .emit();
+                "`#[repr(inttype)]` must be specified for enums with explicit discriminants and non-unit variants"
+            );
+            if let Some(disr_non_unit) = disr_non_unit {
+                err.span_label(
+                    tcx.def_span(disr_non_unit),
+                    "explicit discriminant on non-unit variant specified here",
+                );
+            } else {
+                err.span_label(
+                    tcx.def_span(disr_unit.unwrap()),
+                    "explicit discriminant specified here",
+                );
+                err.span_label(
+                    tcx.def_span(non_unit.unwrap().def_id),
+                    "non-unit discriminant declared here",
+                );
+            }
+            err.emit();
         }
     }
 
diff --git a/compiler/rustc_hir_analysis/src/check/wfcheck.rs b/compiler/rustc_hir_analysis/src/check/wfcheck.rs
index 14ec82ede1c..a62efed13bc 100644
--- a/compiler/rustc_hir_analysis/src/check/wfcheck.rs
+++ b/compiler/rustc_hir_analysis/src/check/wfcheck.rs
@@ -1180,12 +1180,13 @@ fn check_item_fn(
 }
 
 #[instrument(level = "debug", skip(tcx))]
-pub(super) fn check_static_item(
-    tcx: TyCtxt<'_>,
+pub(crate) fn check_static_item<'tcx>(
+    tcx: TyCtxt<'tcx>,
     item_id: LocalDefId,
+    ty: Ty<'tcx>,
+    should_check_for_sync: bool,
 ) -> Result<(), ErrorGuaranteed> {
     enter_wf_checking_ctxt(tcx, item_id, |wfcx| {
-        let ty = tcx.type_of(item_id).instantiate_identity();
         let span = tcx.ty_span(item_id);
         let item_ty = wfcx.deeply_normalize(span, Some(WellFormedLoc::Ty(item_id)), ty);
 
@@ -1212,9 +1213,9 @@ pub(super) fn check_static_item(
         }
 
         // Ensure that the end result is `Sync` in a non-thread local `static`.
-        let should_check_for_sync = tcx.static_mutability(item_id.to_def_id())
-            == Some(hir::Mutability::Not)
+        let should_check_for_sync = should_check_for_sync
             && !is_foreign_item
+            && tcx.static_mutability(item_id.to_def_id()) == Some(hir::Mutability::Not)
             && !tcx.is_thread_local_static(item_id.to_def_id());
 
         if should_check_for_sync {
diff --git a/compiler/rustc_hir_analysis/src/collect/type_of.rs b/compiler/rustc_hir_analysis/src/collect/type_of.rs
index 902a2e15dff..68a91212e50 100644
--- a/compiler/rustc_hir_analysis/src/collect/type_of.rs
+++ b/compiler/rustc_hir_analysis/src/collect/type_of.rs
@@ -14,6 +14,7 @@ use rustc_middle::{bug, span_bug};
 use rustc_span::{DUMMY_SP, Ident, Span};
 
 use super::{HirPlaceholderCollector, ItemCtxt, bad_placeholder};
+use crate::check::wfcheck::check_static_item;
 use crate::errors::TypeofReservedKeywordUsed;
 use crate::hir_ty_lowering::HirTyLowerer;
 
@@ -217,7 +218,13 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::EarlyBinder<'_
                         "static variable",
                     )
                 } else {
-                    icx.lower_ty(ty)
+                    let ty = icx.lower_ty(ty);
+                    // MIR relies on references to statics being scalars.
+                    // Verify that here to avoid ill-formed MIR.
+                    match check_static_item(tcx, def_id, ty, false) {
+                        Ok(()) => ty,
+                        Err(guar) => Ty::new_error(tcx, guar),
+                    }
                 }
             }
             ItemKind::Const(ident, _, ty, body_id) => {
@@ -275,7 +282,15 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::EarlyBinder<'_
                 let args = ty::GenericArgs::identity_for_item(tcx, def_id);
                 Ty::new_fn_def(tcx, def_id.to_def_id(), args)
             }
-            ForeignItemKind::Static(t, _, _) => icx.lower_ty(t),
+            ForeignItemKind::Static(ty, _, _) => {
+                let ty = icx.lower_ty(ty);
+                // MIR relies on references to statics being scalars.
+                // Verify that here to avoid ill-formed MIR.
+                match check_static_item(tcx, def_id, ty, false) {
+                    Ok(()) => ty,
+                    Err(guar) => Ty::new_error(tcx, guar),
+                }
+            }
             ForeignItemKind::Type => Ty::new_foreign(tcx, def_id.to_def_id()),
         },
 
diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/lint.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/lint.rs
index 9abae33ffdb..646ff3ca08d 100644
--- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/lint.rs
+++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/lint.rs
@@ -447,17 +447,30 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
     fn maybe_suggest_assoc_ty_bound(&self, self_ty: &hir::Ty<'_>, diag: &mut Diag<'_>) {
         let mut parents = self.tcx().hir_parent_iter(self_ty.hir_id);
 
-        if let Some((_, hir::Node::AssocItemConstraint(constraint))) = parents.next()
+        if let Some((c_hir_id, hir::Node::AssocItemConstraint(constraint))) = parents.next()
             && let Some(obj_ty) = constraint.ty()
+            && let Some((_, hir::Node::TraitRef(trait_ref))) = parents.next()
         {
-            if let Some((_, hir::Node::TraitRef(..))) = parents.next()
-                && let Some((_, hir::Node::Ty(ty))) = parents.next()
+            if let Some((_, hir::Node::Ty(ty))) = parents.next()
                 && let hir::TyKind::TraitObject(..) = ty.kind
             {
                 // Assoc ty bounds aren't permitted inside trait object types.
                 return;
             }
 
+            if trait_ref
+                .path
+                .segments
+                .iter()
+                .find_map(|seg| {
+                    seg.args.filter(|args| args.constraints.iter().any(|c| c.hir_id == c_hir_id))
+                })
+                .is_none_or(|args| args.parenthesized != hir::GenericArgsParentheses::No)
+            {
+                // Only consider angle-bracketed args (where we have a `=` to replace with `:`).
+                return;
+            }
+
             let lo = if constraint.gen_args.span_ext.is_dummy() {
                 constraint.ident.span
             } else {
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
index dd6eb73a3a0..d18f4e03d2f 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
@@ -1302,8 +1302,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 None => ".clone()".to_string(),
             };
 
+            let span = expr.span.find_oldest_ancestor_in_same_ctxt().shrink_to_hi();
+
             diag.span_suggestion_verbose(
-                expr.span.shrink_to_hi(),
+                span,
                 "consider using clone here",
                 suggestion,
                 Applicability::MachineApplicable,
diff --git a/compiler/rustc_hir_typeck/src/method/suggest.rs b/compiler/rustc_hir_typeck/src/method/suggest.rs
index dbfa7e6273c..e64af8fb7b3 100644
--- a/compiler/rustc_hir_typeck/src/method/suggest.rs
+++ b/compiler/rustc_hir_typeck/src/method/suggest.rs
@@ -264,6 +264,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     err.span_label(within_macro_span, "due to this macro variable");
                 }
                 self.suggest_valid_traits(&mut err, item_name, out_of_scope_traits, true);
+                self.suggest_unwrapping_inner_self(&mut err, source, rcvr_ty, item_name);
                 err.emit()
             }
 
diff --git a/compiler/rustc_hir_typeck/src/upvar.rs b/compiler/rustc_hir_typeck/src/upvar.rs
index be774106abf..df38c3a1214 100644
--- a/compiler/rustc_hir_typeck/src/upvar.rs
+++ b/compiler/rustc_hir_typeck/src/upvar.rs
@@ -1047,7 +1047,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             }
                         }
                     }
-                    lint.note("for more information, see <https://doc.rust-lang.org/nightly/edition-guide/rust-2021/disjoint-capture-in-closures.html>");
+                    lint.note("for more information, see <https://doc.rust-lang.org/edition-guide/rust-2021/disjoint-capture-in-closures.html>");
 
                     let diagnostic_msg = format!(
                         "add a dummy let to cause {migrated_variables_concat} to be fully captured"
diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs
index fb6897c7d89..057fbe2fc4e 100644
--- a/compiler/rustc_interface/src/passes.rs
+++ b/compiler/rustc_interface/src/passes.rs
@@ -208,6 +208,10 @@ fn configure_and_expand(
         // Expand macros now!
         let krate = sess.time("expand_crate", || ecx.monotonic_expander().expand_crate(krate));
 
+        if ecx.nb_macro_errors > 0 {
+            sess.dcx().abort_if_errors();
+        }
+
         // The rest is error reporting and stats
 
         sess.psess.buffered_lints.with_lock(|buffered_lints: &mut Vec<BufferedEarlyLint>| {
diff --git a/compiler/rustc_lint/messages.ftl b/compiler/rustc_lint/messages.ftl
index 1a1cfc9fa6f..69fe7fe83ff 100644
--- a/compiler/rustc_lint/messages.ftl
+++ b/compiler/rustc_lint/messages.ftl
@@ -593,7 +593,7 @@ lint_non_camel_case_type = {$sort} `{$name}` should have an upper camel case nam
 
 lint_non_fmt_panic = panic message is not a string literal
     .note = this usage of `{$name}!()` is deprecated; it will be a hard error in Rust 2021
-    .more_info_note = for more information, see <https://doc.rust-lang.org/nightly/edition-guide/rust-2021/panic-macro-consistency.html>
+    .more_info_note = for more information, see <https://doc.rust-lang.org/edition-guide/rust-2021/panic-macro-consistency.html>
     .supports_fmt_note = the `{$name}!()` macro supports formatting, so there's no need for the `format!()` macro here
     .supports_fmt_suggestion = remove the `format!(..)` macro call
     .display_suggestion = add a "{"{"}{"}"}" format string to `Display` the message
diff --git a/compiler/rustc_lint/src/builtin.rs b/compiler/rustc_lint/src/builtin.rs
index 73e68834232..bf128b5914f 100644
--- a/compiler/rustc_lint/src/builtin.rs
+++ b/compiler/rustc_lint/src/builtin.rs
@@ -1654,7 +1654,7 @@ declare_lint! {
     "`...` range patterns are deprecated",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2021),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2021/warnings-promoted-to-error.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2021/warnings-promoted-to-error.html>",
     };
 }
 
@@ -1835,7 +1835,7 @@ declare_lint! {
     "detects edition keywords being used as an identifier",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/gen-keyword.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/gen-keyword.html>",
     };
 }
 
@@ -2870,7 +2870,7 @@ impl<'tcx> LateLintPass<'tcx> for AsmLabels {
         if let hir::Expr {
             kind:
                 hir::ExprKind::InlineAsm(hir::InlineAsm {
-                    asm_macro: AsmMacro::Asm | AsmMacro::NakedAsm,
+                    asm_macro: asm_macro @ (AsmMacro::Asm | AsmMacro::NakedAsm),
                     template_strs,
                     options,
                     ..
@@ -2878,6 +2878,15 @@ impl<'tcx> LateLintPass<'tcx> for AsmLabels {
             ..
         } = expr
         {
+            // Non-generic naked functions are allowed to define arbitrary
+            // labels.
+            if *asm_macro == AsmMacro::NakedAsm {
+                let def_id = expr.hir_id.owner.def_id;
+                if !cx.tcx.generics_of(def_id).requires_monomorphization(cx.tcx) {
+                    return;
+                }
+            }
+
             // asm with `options(raw)` does not do replacement with `{` and `}`.
             let raw = options.contains(InlineAsmOptions::RAW);
 
diff --git a/compiler/rustc_lint/src/if_let_rescope.rs b/compiler/rustc_lint/src/if_let_rescope.rs
index 263ea6fa070..ff67ed1bc55 100644
--- a/compiler/rustc_lint/src/if_let_rescope.rs
+++ b/compiler/rustc_lint/src/if_let_rescope.rs
@@ -87,7 +87,7 @@ declare_lint! {
     rewriting in `match` is an option to preserve the semantics up to Edition 2021",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionSemanticsChange(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/temporary-if-let-scope.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/temporary-if-let-scope.html>",
     };
 }
 
diff --git a/compiler/rustc_lint/src/impl_trait_overcaptures.rs b/compiler/rustc_lint/src/impl_trait_overcaptures.rs
index c17281deff4..b9afb62cf1c 100644
--- a/compiler/rustc_lint/src/impl_trait_overcaptures.rs
+++ b/compiler/rustc_lint/src/impl_trait_overcaptures.rs
@@ -71,7 +71,7 @@ declare_lint! {
     "`impl Trait` will capture more lifetimes than possibly intended in edition 2024",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionSemanticsChange(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/rpit-lifetime-capture.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/rpit-lifetime-capture.html>",
     };
 }
 
diff --git a/compiler/rustc_lint/src/late.rs b/compiler/rustc_lint/src/late.rs
index c681deea779..ccfba715a1b 100644
--- a/compiler/rustc_lint/src/late.rs
+++ b/compiler/rustc_lint/src/late.rs
@@ -356,7 +356,16 @@ pub fn late_lint_mod<'tcx, T: LateLintPass<'tcx> + 'tcx>(
     let store = unerased_lint_store(tcx.sess);
 
     if store.late_module_passes.is_empty() {
-        late_lint_mod_inner(tcx, module_def_id, context, builtin_lints);
+        // If all builtin lints can be skipped, there is no point in running `late_lint_mod_inner`
+        // at all. This happens often for dependencies built with `--cap-lints=allow`.
+        let dont_need_to_run = tcx.lints_that_dont_need_to_run(());
+        let can_skip_lints = builtin_lints
+            .get_lints()
+            .iter()
+            .all(|lint| dont_need_to_run.contains(&LintId::of(lint)));
+        if !can_skip_lints {
+            late_lint_mod_inner(tcx, module_def_id, context, builtin_lints);
+        }
     } else {
         let builtin_lints = Box::new(builtin_lints) as Box<dyn LateLintPass<'tcx>>;
         let mut binding = store
diff --git a/compiler/rustc_lint/src/levels.rs b/compiler/rustc_lint/src/levels.rs
index 16eeb89207b..ac47897b568 100644
--- a/compiler/rustc_lint/src/levels.rs
+++ b/compiler/rustc_lint/src/levels.rs
@@ -933,6 +933,7 @@ impl<'s, P: LintLevelsProvider> LintLevelsBuilder<'s, P> {
     fn check_gated_lint(&self, lint_id: LintId, span: Span, lint_from_cli: bool) -> bool {
         let feature = if let Some(feature) = lint_id.lint.feature_gate
             && !self.features.enabled(feature)
+            && !span.allows_unstable(feature)
         {
             // Lint is behind a feature that is not enabled; eventually return false.
             feature
diff --git a/compiler/rustc_lint/src/macro_expr_fragment_specifier_2024_migration.rs b/compiler/rustc_lint/src/macro_expr_fragment_specifier_2024_migration.rs
index ce280fef8b6..7de6fbd941b 100644
--- a/compiler/rustc_lint/src/macro_expr_fragment_specifier_2024_migration.rs
+++ b/compiler/rustc_lint/src/macro_expr_fragment_specifier_2024_migration.rs
@@ -65,7 +65,7 @@ declare_lint! {
     /// to ensure the macros implement the desired behavior.
     ///
     /// [editions]: https://doc.rust-lang.org/edition-guide/
-    /// [macro matcher fragment specifiers]: https://doc.rust-lang.org/nightly/edition-guide/rust-2024/macro-fragment-specifiers.html
+    /// [macro matcher fragment specifiers]: https://doc.rust-lang.org/edition-guide/rust-2024/macro-fragment-specifiers.html
     /// [`cargo fix`]: https://doc.rust-lang.org/cargo/commands/cargo-fix.html
     pub EDITION_2024_EXPR_FRAGMENT_SPECIFIER,
     Allow,
@@ -73,7 +73,7 @@ declare_lint! {
     To keep the existing behavior, use the `expr_2021` fragment specifier.",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionSemanticsChange(Edition::Edition2024),
-        reference: "Migration Guide <https://doc.rust-lang.org/nightly/edition-guide/rust-2024/macro-fragment-specifiers.html>",
+        reference: "Migration Guide <https://doc.rust-lang.org/edition-guide/rust-2024/macro-fragment-specifiers.html>",
     };
 }
 
diff --git a/compiler/rustc_lint/src/passes.rs b/compiler/rustc_lint/src/passes.rs
index affea1b80ec..191eb721b34 100644
--- a/compiler/rustc_lint/src/passes.rs
+++ b/compiler/rustc_lint/src/passes.rs
@@ -92,7 +92,7 @@ macro_rules! expand_combined_late_lint_pass_methods {
 /// Combines multiple lints passes into a single lint pass, at compile time,
 /// for maximum speed. Each `check_foo` method in `$methods` within this pass
 /// simply calls `check_foo` once per `$pass`. Compare with
-/// `LateLintPassObjects`, which is similar, but combines lint passes at
+/// `RuntimeCombinedLateLintPass`, which is similar, but combines lint passes at
 /// runtime.
 #[macro_export]
 macro_rules! declare_combined_late_lint_pass {
@@ -123,10 +123,10 @@ macro_rules! declare_combined_late_lint_pass {
         #[allow(rustc::lint_pass_impl_without_macro)]
         impl $crate::LintPass for $name {
             fn name(&self) -> &'static str {
-                panic!()
+                stringify!($name)
             }
             fn get_lints(&self) -> LintVec {
-                panic!()
+                $name::get_lints()
             }
         }
     )
diff --git a/compiler/rustc_lint/src/shadowed_into_iter.rs b/compiler/rustc_lint/src/shadowed_into_iter.rs
index 00fa0499556..d296ae46f43 100644
--- a/compiler/rustc_lint/src/shadowed_into_iter.rs
+++ b/compiler/rustc_lint/src/shadowed_into_iter.rs
@@ -32,7 +32,7 @@ declare_lint! {
     "detects calling `into_iter` on arrays in Rust 2015 and 2018",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionSemanticsChange(Edition::Edition2021),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2021/IntoIterator-for-arrays.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2021/IntoIterator-for-arrays.html>",
     };
 }
 
@@ -61,7 +61,7 @@ declare_lint! {
     "detects calling `into_iter` on boxed slices in Rust 2015, 2018, and 2021",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionSemanticsChange(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/intoiterator-box-slice.html>"
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/intoiterator-box-slice.html>"
     };
 }
 
diff --git a/compiler/rustc_lint/src/static_mut_refs.rs b/compiler/rustc_lint/src/static_mut_refs.rs
index 4dda3c7951b..16e1fb0192b 100644
--- a/compiler/rustc_lint/src/static_mut_refs.rs
+++ b/compiler/rustc_lint/src/static_mut_refs.rs
@@ -54,7 +54,7 @@ declare_lint! {
     "creating a shared reference to mutable static",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/static-mut-references.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/static-mut-references.html>",
         explain_reason: false,
     };
     @edition Edition2024 => Deny;
diff --git a/compiler/rustc_lint/src/unused.rs b/compiler/rustc_lint/src/unused.rs
index a9eb1739f7f..df9f3a500d9 100644
--- a/compiler/rustc_lint/src/unused.rs
+++ b/compiler/rustc_lint/src/unused.rs
@@ -1340,7 +1340,15 @@ impl EarlyLintPass for UnusedParens {
                 self.with_self_ty_parens = false;
             }
             ast::TyKind::Ref(_, mut_ty) | ast::TyKind::Ptr(mut_ty) => {
-                self.in_no_bounds_pos.insert(mut_ty.ty.id, NoBoundsException::OneBound);
+                // If this type itself appears in no-bounds position, we propagate its
+                // potentially tighter constraint or risk a false posive (issue 143653).
+                let own_constraint = self.in_no_bounds_pos.get(&ty.id);
+                let constraint = match own_constraint {
+                    Some(NoBoundsException::None) => NoBoundsException::None,
+                    Some(NoBoundsException::OneBound) => NoBoundsException::OneBound,
+                    None => NoBoundsException::OneBound,
+                };
+                self.in_no_bounds_pos.insert(mut_ty.ty.id, constraint);
             }
             ast::TyKind::TraitObject(bounds, _) | ast::TyKind::ImplTrait(_, bounds) => {
                 for i in 0..bounds.len() {
diff --git a/compiler/rustc_lint_defs/src/builtin.rs b/compiler/rustc_lint_defs/src/builtin.rs
index a08d68e2d15..b1edb5c3044 100644
--- a/compiler/rustc_lint_defs/src/builtin.rs
+++ b/compiler/rustc_lint_defs/src/builtin.rs
@@ -1814,7 +1814,7 @@ declare_lint! {
     "suggest using `dyn Trait` for trait objects",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2021),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2021/warnings-promoted-to-error.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2021/warnings-promoted-to-error.html>",
     };
 }
 
@@ -2472,7 +2472,7 @@ declare_lint! {
     "unsafe operations in unsafe functions without an explicit unsafe block are deprecated",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionSemanticsChange(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/unsafe-op-in-unsafe-fn.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/unsafe-op-in-unsafe-fn.html>",
         explain_reason: false
     };
     @edition Edition2024 => Warn;
@@ -3445,7 +3445,7 @@ declare_lint! {
     "detects usage of old versions of or-patterns",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2021),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2021/or-patterns-macro-rules.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2021/or-patterns-macro-rules.html>",
     };
 }
 
@@ -3494,7 +3494,7 @@ declare_lint! {
         prelude in future editions",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2021),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2021/prelude.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2021/prelude.html>",
     };
 }
 
@@ -3534,7 +3534,7 @@ declare_lint! {
         prelude in future editions",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/prelude.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/prelude.html>",
     };
 }
 
@@ -3571,7 +3571,7 @@ declare_lint! {
     "identifiers that will be parsed as a prefix in Rust 2021",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2021),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2021/reserving-syntax.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2021/reserving-syntax.html>",
     };
     crate_level_only
 }
@@ -4100,7 +4100,7 @@ declare_lint! {
     "never type fallback affecting unsafe function calls",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionAndFutureReleaseSemanticsChange(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/never-type-fallback.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/never-type-fallback.html>",
         report_in_deps: true,
     };
     @edition Edition2024 => Deny;
@@ -4155,7 +4155,7 @@ declare_lint! {
     "never type fallback affecting unsafe function calls",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionAndFutureReleaseError(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/never-type-fallback.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/never-type-fallback.html>",
         report_in_deps: true,
     };
     report_in_external_macro
@@ -4740,7 +4740,7 @@ declare_lint! {
     "detects unsafe functions being used as safe functions",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/newly-unsafe-functions.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/newly-unsafe-functions.html>",
     };
 }
 
@@ -4776,7 +4776,7 @@ declare_lint! {
     "detects missing unsafe keyword on extern declarations",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/unsafe-extern.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/unsafe-extern.html>",
     };
 }
 
@@ -4817,7 +4817,7 @@ declare_lint! {
     "detects unsafe attributes outside of unsafe",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/unsafe-attributes.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/unsafe-attributes.html>",
     };
 }
 
@@ -5014,7 +5014,7 @@ declare_lint! {
     "Detect and warn on significant change in drop order in tail expression location",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionSemanticsChange(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/temporary-tail-expr-scope.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/temporary-tail-expr-scope.html>",
     };
 }
 
@@ -5053,7 +5053,7 @@ declare_lint! {
     "will be parsed as a guarded string in Rust 2024",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/reserved-syntax.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/reserved-syntax.html>",
     };
     crate_level_only
 }
diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
index 82568ed4ae1..c9814beedd6 100644
--- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
@@ -1610,7 +1610,7 @@ extern "C" void LLVMRustPositionBefore(LLVMBuilderRef B, LLVMValueRef Instr) {
 
 extern "C" void LLVMRustPositionAfter(LLVMBuilderRef B, LLVMValueRef Instr) {
   if (auto I = dyn_cast<Instruction>(unwrap<Value>(Instr))) {
-    auto J = I->getNextNonDebugInstruction();
+    auto J = I->getNextNode();
     unwrap(B)->SetInsertPoint(J);
   }
 }
diff --git a/compiler/rustc_metadata/messages.ftl b/compiler/rustc_metadata/messages.ftl
index 3bef5ca114b..4d3e879a098 100644
--- a/compiler/rustc_metadata/messages.ftl
+++ b/compiler/rustc_metadata/messages.ftl
@@ -330,3 +330,6 @@ metadata_wasm_import_form =
 
 metadata_whole_archive_needs_static =
     linking modifier `whole-archive` is only compatible with `static` linking kind
+
+metadata_raw_dylib_malformed =
+    link name must be well-formed if link kind is `raw-dylib`
diff --git a/compiler/rustc_metadata/src/errors.rs b/compiler/rustc_metadata/src/errors.rs
index 4a3b43167cf..0332dba1077 100644
--- a/compiler/rustc_metadata/src/errors.rs
+++ b/compiler/rustc_metadata/src/errors.rs
@@ -815,3 +815,10 @@ pub struct AsyncDropTypesInDependency {
     pub extern_crate: Symbol,
     pub local_crate: Symbol,
 }
+
+#[derive(Diagnostic)]
+#[diag(metadata_raw_dylib_malformed)]
+pub struct RawDylibMalformed {
+    #[primary_span]
+    pub span: Span,
+}
diff --git a/compiler/rustc_metadata/src/native_libs.rs b/compiler/rustc_metadata/src/native_libs.rs
index 4d276f814ef..ed0f084ea83 100644
--- a/compiler/rustc_metadata/src/native_libs.rs
+++ b/compiler/rustc_metadata/src/native_libs.rs
@@ -700,8 +700,21 @@ impl<'tcx> Collector<'tcx> {
             .link_ordinal
             .map_or(import_name_type, |ord| Some(PeImportNameType::Ordinal(ord)));
 
+        let name = codegen_fn_attrs.link_name.unwrap_or_else(|| self.tcx.item_name(item));
+
+        if self.tcx.sess.target.binary_format == BinaryFormat::Elf {
+            let name = name.as_str();
+            if name.contains('\0') {
+                self.tcx.dcx().emit_err(errors::RawDylibMalformed { span });
+            } else if let Some((left, right)) = name.split_once('@')
+                && (left.is_empty() || right.is_empty() || right.contains('@'))
+            {
+                self.tcx.dcx().emit_err(errors::RawDylibMalformed { span });
+            }
+        }
+
         DllImport {
-            name: codegen_fn_attrs.link_name.unwrap_or_else(|| self.tcx.item_name(item)),
+            name,
             import_name_type,
             calling_convention,
             span,
diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs
index 5cd98038fc6..24a3a6e0c4f 100644
--- a/compiler/rustc_metadata/src/rmeta/encoder.rs
+++ b/compiler/rustc_metadata/src/rmeta/encoder.rs
@@ -2124,11 +2124,11 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             };
             let def_id = id.owner_id.to_def_id();
 
-            self.tables.defaultness.set_some(def_id.index, tcx.defaultness(def_id));
-
             if of_trait && let Some(header) = tcx.impl_trait_header(def_id) {
                 record!(self.tables.impl_trait_header[def_id] <- header);
 
+                self.tables.defaultness.set_some(def_id.index, tcx.defaultness(def_id));
+
                 let trait_ref = header.trait_ref.instantiate_identity();
                 let simplified_self_ty = fast_reject::simplify_type(
                     self.tcx,
diff --git a/compiler/rustc_middle/src/hooks/mod.rs b/compiler/rustc_middle/src/hooks/mod.rs
index c5ce6efcb81..9d2f0a45237 100644
--- a/compiler/rustc_middle/src/hooks/mod.rs
+++ b/compiler/rustc_middle/src/hooks/mod.rs
@@ -50,10 +50,10 @@ macro_rules! declare_hooks {
 declare_hooks! {
     /// Tries to destructure an `mir::Const` ADT or array into its variant index
     /// and its field values. This should only be used for pretty printing.
-    hook try_destructure_mir_constant_for_user_output(val: mir::ConstValue<'tcx>, ty: Ty<'tcx>) -> Option<mir::DestructuredConstant<'tcx>>;
+    hook try_destructure_mir_constant_for_user_output(val: mir::ConstValue, ty: Ty<'tcx>) -> Option<mir::DestructuredConstant<'tcx>>;
 
     /// Getting a &core::panic::Location referring to a span.
-    hook const_caller_location(file: rustc_span::Symbol, line: u32, col: u32) -> mir::ConstValue<'tcx>;
+    hook const_caller_location(file: rustc_span::Symbol, line: u32, col: u32) -> mir::ConstValue;
 
     /// Returns `true` if this def is a function-like thing that is eligible for
     /// coverage instrumentation under `-Cinstrument-coverage`.
diff --git a/compiler/rustc_middle/src/mir/consts.rs b/compiler/rustc_middle/src/mir/consts.rs
index fb941977528..96131d47a17 100644
--- a/compiler/rustc_middle/src/mir/consts.rs
+++ b/compiler/rustc_middle/src/mir/consts.rs
@@ -9,9 +9,7 @@ use rustc_span::{DUMMY_SP, Span, Symbol};
 use rustc_type_ir::TypeVisitableExt;
 
 use super::interpret::ReportedErrorInfo;
-use crate::mir::interpret::{
-    AllocId, AllocRange, ConstAllocation, ErrorHandled, GlobalAlloc, Scalar, alloc_range,
-};
+use crate::mir::interpret::{AllocId, AllocRange, ErrorHandled, GlobalAlloc, Scalar, alloc_range};
 use crate::mir::{Promoted, pretty_print_const_value};
 use crate::ty::print::{pretty_print_const, with_no_trimmed_paths};
 use crate::ty::{self, ConstKind, GenericArgsRef, ScalarInt, Ty, TyCtxt};
@@ -33,8 +31,8 @@ pub struct ConstAlloc<'tcx> {
 /// Represents a constant value in Rust. `Scalar` and `Slice` are optimizations for
 /// array length computations, enum discriminants and the pattern matching logic.
 #[derive(Copy, Clone, Debug, Eq, PartialEq, TyEncodable, TyDecodable, Hash)]
-#[derive(HashStable, Lift)]
-pub enum ConstValue<'tcx> {
+#[derive(HashStable)]
+pub enum ConstValue {
     /// Used for types with `layout::abi::Scalar` ABI.
     ///
     /// Not using the enum `Value` to encode that this must not be `Uninit`.
@@ -52,7 +50,7 @@ pub enum ConstValue<'tcx> {
     Slice {
         /// The allocation storing the slice contents.
         /// This always points to the beginning of the allocation.
-        data: ConstAllocation<'tcx>,
+        alloc_id: AllocId,
         /// The metadata field of the reference.
         /// This is a "target usize", so we use `u64` as in the interpreter.
         meta: u64,
@@ -75,9 +73,9 @@ pub enum ConstValue<'tcx> {
 }
 
 #[cfg(target_pointer_width = "64")]
-rustc_data_structures::static_assert_size!(ConstValue<'_>, 24);
+rustc_data_structures::static_assert_size!(ConstValue, 24);
 
-impl<'tcx> ConstValue<'tcx> {
+impl ConstValue {
     #[inline]
     pub fn try_to_scalar(&self) -> Option<Scalar> {
         match *self {
@@ -98,11 +96,11 @@ impl<'tcx> ConstValue<'tcx> {
         self.try_to_scalar_int()?.try_into().ok()
     }
 
-    pub fn try_to_target_usize(&self, tcx: TyCtxt<'tcx>) -> Option<u64> {
+    pub fn try_to_target_usize(&self, tcx: TyCtxt<'_>) -> Option<u64> {
         Some(self.try_to_scalar_int()?.to_target_usize(tcx))
     }
 
-    pub fn try_to_bits_for_ty(
+    pub fn try_to_bits_for_ty<'tcx>(
         &self,
         tcx: TyCtxt<'tcx>,
         typing_env: ty::TypingEnv<'tcx>,
@@ -132,12 +130,15 @@ impl<'tcx> ConstValue<'tcx> {
     }
 
     /// Must only be called on constants of type `&str` or `&[u8]`!
-    pub fn try_get_slice_bytes_for_diagnostics(&self, tcx: TyCtxt<'tcx>) -> Option<&'tcx [u8]> {
-        let (data, start, end) = match self {
+    pub fn try_get_slice_bytes_for_diagnostics<'tcx>(
+        &self,
+        tcx: TyCtxt<'tcx>,
+    ) -> Option<&'tcx [u8]> {
+        let (alloc_id, start, len) = match self {
             ConstValue::Scalar(_) | ConstValue::ZeroSized => {
                 bug!("`try_get_slice_bytes` on non-slice constant")
             }
-            &ConstValue::Slice { data, meta } => (data, 0, meta),
+            &ConstValue::Slice { alloc_id, meta } => (alloc_id, 0, meta),
             &ConstValue::Indirect { alloc_id, offset } => {
                 // The reference itself is stored behind an indirection.
                 // Load the reference, and then load the actual slice contents.
@@ -170,26 +171,29 @@ impl<'tcx> ConstValue<'tcx> {
                 // Non-empty slice, must have memory. We know this is a relative pointer.
                 let (inner_prov, offset) =
                     ptr.into_pointer_or_addr().ok()?.prov_and_relative_offset();
-                let data = tcx.global_alloc(inner_prov.alloc_id()).unwrap_memory();
-                (data, offset.bytes(), offset.bytes() + len)
+                (inner_prov.alloc_id(), offset.bytes(), len)
             }
         };
 
+        let data = tcx.global_alloc(alloc_id).unwrap_memory();
+
         // This is for diagnostics only, so we are okay to use `inspect_with_uninit_and_ptr_outside_interpreter`.
         let start = start.try_into().unwrap();
-        let end = end.try_into().unwrap();
+        let end = start + usize::try_from(len).unwrap();
         Some(data.inner().inspect_with_uninit_and_ptr_outside_interpreter(start..end))
     }
 
     /// Check if a constant may contain provenance information. This is used by MIR opts.
     /// Can return `true` even if there is no provenance.
-    pub fn may_have_provenance(&self, tcx: TyCtxt<'tcx>, size: Size) -> bool {
+    pub fn may_have_provenance(&self, tcx: TyCtxt<'_>, size: Size) -> bool {
         match *self {
             ConstValue::ZeroSized | ConstValue::Scalar(Scalar::Int(_)) => return false,
             ConstValue::Scalar(Scalar::Ptr(..)) => return true,
             // It's hard to find out the part of the allocation we point to;
             // just conservatively check everything.
-            ConstValue::Slice { data, meta: _ } => !data.inner().provenance().ptrs().is_empty(),
+            ConstValue::Slice { alloc_id, meta: _ } => {
+                !tcx.global_alloc(alloc_id).unwrap_memory().inner().provenance().ptrs().is_empty()
+            }
             ConstValue::Indirect { alloc_id, offset } => !tcx
                 .global_alloc(alloc_id)
                 .unwrap_memory()
@@ -200,7 +204,7 @@ impl<'tcx> ConstValue<'tcx> {
     }
 
     /// Check if a constant only contains uninitialized bytes.
-    pub fn all_bytes_uninit(&self, tcx: TyCtxt<'tcx>) -> bool {
+    pub fn all_bytes_uninit(&self, tcx: TyCtxt<'_>) -> bool {
         let ConstValue::Indirect { alloc_id, .. } = self else {
             return false;
         };
@@ -247,7 +251,7 @@ pub enum Const<'tcx> {
 
     /// This constant cannot go back into the type system, as it represents
     /// something the type system cannot handle (e.g. pointers).
-    Val(ConstValue<'tcx>, Ty<'tcx>),
+    Val(ConstValue, Ty<'tcx>),
 }
 
 impl<'tcx> Const<'tcx> {
@@ -343,7 +347,7 @@ impl<'tcx> Const<'tcx> {
         tcx: TyCtxt<'tcx>,
         typing_env: ty::TypingEnv<'tcx>,
         span: Span,
-    ) -> Result<ConstValue<'tcx>, ErrorHandled> {
+    ) -> Result<ConstValue, ErrorHandled> {
         match self {
             Const::Ty(_, c) => {
                 if c.has_non_region_param() {
@@ -440,7 +444,7 @@ impl<'tcx> Const<'tcx> {
     }
 
     #[inline]
-    pub fn from_value(val: ConstValue<'tcx>, ty: Ty<'tcx>) -> Self {
+    pub fn from_value(val: ConstValue, ty: Ty<'tcx>) -> Self {
         Self::Val(val, ty)
     }
 
@@ -487,9 +491,8 @@ impl<'tcx> Const<'tcx> {
     /// taking into account even pointer identity tests.
     pub fn is_deterministic(&self) -> bool {
         // Some constants may generate fresh allocations for pointers they contain,
-        // so using the same constant twice can yield two different results:
-        // - valtrees purposefully generate new allocations
-        // - ConstValue::Slice also generate new allocations
+        // so using the same constant twice can yield two different results.
+        // Notably, valtrees purposefully generate new allocations.
         match self {
             Const::Ty(_, c) => match c.kind() {
                 ty::ConstKind::Param(..) => true,
@@ -507,11 +510,11 @@ impl<'tcx> Const<'tcx> {
                 | ty::ConstKind::Placeholder(..) => bug!(),
             },
             Const::Unevaluated(..) => false,
-            // If the same slice appears twice in the MIR, we cannot guarantee that we will
-            // give the same `AllocId` to the data.
-            Const::Val(ConstValue::Slice { .. }, _) => false,
             Const::Val(
-                ConstValue::ZeroSized | ConstValue::Scalar(_) | ConstValue::Indirect { .. },
+                ConstValue::Slice { .. }
+                | ConstValue::ZeroSized
+                | ConstValue::Scalar(_)
+                | ConstValue::Indirect { .. },
                 _,
             ) => true,
         }
@@ -574,7 +577,7 @@ impl<'tcx> Display for Const<'tcx> {
 /// Const-related utilities
 
 impl<'tcx> TyCtxt<'tcx> {
-    pub fn span_as_caller_location(self, span: Span) -> ConstValue<'tcx> {
+    pub fn span_as_caller_location(self, span: Span) -> ConstValue {
         let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span);
         let caller = self.sess.source_map().lookup_char_pos(topmost.lo());
         self.const_caller_location(
diff --git a/compiler/rustc_middle/src/mir/interpret/error.rs b/compiler/rustc_middle/src/mir/interpret/error.rs
index 3e68afbfabd..2b0cfb86564 100644
--- a/compiler/rustc_middle/src/mir/interpret/error.rs
+++ b/compiler/rustc_middle/src/mir/interpret/error.rs
@@ -137,7 +137,7 @@ impl<'tcx> ValTreeCreationError<'tcx> {
 
 pub type EvalToAllocationRawResult<'tcx> = Result<ConstAlloc<'tcx>, ErrorHandled>;
 pub type EvalStaticInitializerRawResult<'tcx> = Result<ConstAllocation<'tcx>, ErrorHandled>;
-pub type EvalToConstValueResult<'tcx> = Result<ConstValue<'tcx>, ErrorHandled>;
+pub type EvalToConstValueResult<'tcx> = Result<ConstValue, ErrorHandled>;
 pub type EvalToValTreeResult<'tcx> = Result<ValTree<'tcx>, ValTreeCreationError<'tcx>>;
 
 #[cfg(target_pointer_width = "64")]
diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs
index 2d7ddd105bd..105736b9e24 100644
--- a/compiler/rustc_middle/src/mir/mono.rs
+++ b/compiler/rustc_middle/src/mir/mono.rs
@@ -143,10 +143,8 @@ impl<'tcx> MonoItem<'tcx> {
         };
 
         // Similarly, the executable entrypoint must be instantiated exactly once.
-        if let Some((entry_def_id, _)) = tcx.entry_fn(()) {
-            if instance.def_id() == entry_def_id {
-                return InstantiationMode::GloballyShared { may_conflict: false };
-            }
+        if tcx.is_entrypoint(instance.def_id()) {
+            return InstantiationMode::GloballyShared { may_conflict: false };
         }
 
         // If the function is #[naked] or contains any other attribute that requires exactly-once
diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs
index 8e403dfddae..809cdb329f7 100644
--- a/compiler/rustc_middle/src/mir/pretty.rs
+++ b/compiler/rustc_middle/src/mir/pretty.rs
@@ -1465,7 +1465,7 @@ impl<'tcx> Visitor<'tcx> for ExtraComments<'tcx> {
                 self.push(&format!("+ user_ty: {user_ty:?}"));
             }
 
-            let fmt_val = |val: ConstValue<'tcx>, ty: Ty<'tcx>| {
+            let fmt_val = |val: ConstValue, ty: Ty<'tcx>| {
                 let tcx = self.tcx;
                 rustc_data_structures::make_display(move |fmt| {
                     pretty_print_const_value_tcx(tcx, val, ty, fmt)
@@ -1562,16 +1562,12 @@ pub fn write_allocations<'tcx>(
         alloc.inner().provenance().ptrs().values().map(|p| p.alloc_id())
     }
 
-    fn alloc_id_from_const_val(val: ConstValue<'_>) -> Option<AllocId> {
+    fn alloc_id_from_const_val(val: ConstValue) -> Option<AllocId> {
         match val {
             ConstValue::Scalar(interpret::Scalar::Ptr(ptr, _)) => Some(ptr.provenance.alloc_id()),
             ConstValue::Scalar(interpret::Scalar::Int { .. }) => None,
             ConstValue::ZeroSized => None,
-            ConstValue::Slice { .. } => {
-                // `u8`/`str` slices, shouldn't contain pointers that we want to print.
-                None
-            }
-            ConstValue::Indirect { alloc_id, .. } => {
+            ConstValue::Slice { alloc_id, .. } | ConstValue::Indirect { alloc_id, .. } => {
                 // FIXME: we don't actually want to print all of these, since some are printed nicely directly as values inline in MIR.
                 // Really we'd want `pretty_print_const_value` to decide which allocations to print, instead of having a separate visitor.
                 Some(alloc_id)
@@ -1885,7 +1881,7 @@ fn pretty_print_byte_str(fmt: &mut Formatter<'_>, byte_str: &[u8]) -> fmt::Resul
 fn comma_sep<'tcx>(
     tcx: TyCtxt<'tcx>,
     fmt: &mut Formatter<'_>,
-    elems: Vec<(ConstValue<'tcx>, Ty<'tcx>)>,
+    elems: Vec<(ConstValue, Ty<'tcx>)>,
 ) -> fmt::Result {
     let mut first = true;
     for (ct, ty) in elems {
@@ -1900,7 +1896,7 @@ fn comma_sep<'tcx>(
 
 fn pretty_print_const_value_tcx<'tcx>(
     tcx: TyCtxt<'tcx>,
-    ct: ConstValue<'tcx>,
+    ct: ConstValue,
     ty: Ty<'tcx>,
     fmt: &mut Formatter<'_>,
 ) -> fmt::Result {
@@ -1947,7 +1943,7 @@ fn pretty_print_const_value_tcx<'tcx>(
             let ct = tcx.lift(ct).unwrap();
             let ty = tcx.lift(ty).unwrap();
             if let Some(contents) = tcx.try_destructure_mir_constant_for_user_output(ct, ty) {
-                let fields: Vec<(ConstValue<'_>, Ty<'_>)> = contents.fields.to_vec();
+                let fields: Vec<(ConstValue, Ty<'_>)> = contents.fields.to_vec();
                 match *ty.kind() {
                     ty::Array(..) => {
                         fmt.write_str("[")?;
@@ -2028,7 +2024,7 @@ fn pretty_print_const_value_tcx<'tcx>(
 }
 
 pub(crate) fn pretty_print_const_value<'tcx>(
-    ct: ConstValue<'tcx>,
+    ct: ConstValue,
     ty: Ty<'tcx>,
     fmt: &mut Formatter<'_>,
 ) -> fmt::Result {
diff --git a/compiler/rustc_middle/src/mir/query.rs b/compiler/rustc_middle/src/mir/query.rs
index 3fc05f2caf2..a8a95c699d8 100644
--- a/compiler/rustc_middle/src/mir/query.rs
+++ b/compiler/rustc_middle/src/mir/query.rs
@@ -173,5 +173,5 @@ pub enum AnnotationSource {
 #[derive(Copy, Clone, Debug, HashStable)]
 pub struct DestructuredConstant<'tcx> {
     pub variant: Option<VariantIdx>,
-    pub fields: &'tcx [(ConstValue<'tcx>, Ty<'tcx>)],
+    pub fields: &'tcx [(ConstValue, Ty<'tcx>)],
 }
diff --git a/compiler/rustc_middle/src/query/erase.rs b/compiler/rustc_middle/src/query/erase.rs
index f138c5ca039..dab5900b4ab 100644
--- a/compiler/rustc_middle/src/query/erase.rs
+++ b/compiler/rustc_middle/src/query/erase.rs
@@ -153,8 +153,8 @@ impl EraseType for Result<mir::ConstAlloc<'_>, mir::interpret::ErrorHandled> {
     type Result = [u8; size_of::<Result<mir::ConstAlloc<'static>, mir::interpret::ErrorHandled>>()];
 }
 
-impl EraseType for Result<mir::ConstValue<'_>, mir::interpret::ErrorHandled> {
-    type Result = [u8; size_of::<Result<mir::ConstValue<'static>, mir::interpret::ErrorHandled>>()];
+impl EraseType for Result<mir::ConstValue, mir::interpret::ErrorHandled> {
+    type Result = [u8; size_of::<Result<mir::ConstValue, mir::interpret::ErrorHandled>>()];
 }
 
 impl EraseType for EvalToValTreeResult<'_> {
@@ -301,6 +301,7 @@ trivial! {
     rustc_middle::middle::resolve_bound_vars::ResolvedArg,
     rustc_middle::middle::stability::DeprecationEntry,
     rustc_middle::mir::ConstQualifs,
+    rustc_middle::mir::ConstValue,
     rustc_middle::mir::interpret::AllocId,
     rustc_middle::mir::interpret::CtfeProvenance,
     rustc_middle::mir::interpret::ErrorHandled,
@@ -362,7 +363,6 @@ tcx_lifetime! {
     rustc_middle::mir::Const,
     rustc_middle::mir::DestructuredConstant,
     rustc_middle::mir::ConstAlloc,
-    rustc_middle::mir::ConstValue,
     rustc_middle::mir::interpret::GlobalId,
     rustc_middle::mir::interpret::LitToConstInput,
     rustc_middle::mir::interpret::EvalStaticInitializerRawResult,
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
index ad7f4973e23..b0d579a546f 100644
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@@ -1363,7 +1363,7 @@ rustc_queries! {
     }
 
     /// Converts a type-level constant value into a MIR constant value.
-    query valtree_to_const_val(key: ty::Value<'tcx>) -> mir::ConstValue<'tcx> {
+    query valtree_to_const_val(key: ty::Value<'tcx>) -> mir::ConstValue {
         desc { "converting type-level constant value to MIR constant value"}
     }
 
@@ -2152,9 +2152,6 @@ rustc_queries! {
         desc { |tcx| "collecting child items of module `{}`", tcx.def_path_str(def_id) }
         separate_provide_extern
     }
-    query extern_mod_stmt_cnum(def_id: LocalDefId) -> Option<CrateNum> {
-        desc { |tcx| "computing crate imported by `{}`", tcx.def_path_str(def_id) }
-    }
 
     /// Gets the number of definitions in a foreign crate.
     ///
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index 51db92ecd78..6f21160d1f6 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -5,7 +5,7 @@
 pub mod tls;
 
 use std::assert_matches::debug_assert_matches;
-use std::borrow::Borrow;
+use std::borrow::{Borrow, Cow};
 use std::cmp::Ordering;
 use std::env::VarError;
 use std::ffi::OsStr;
@@ -1041,11 +1041,13 @@ const NUM_PREINTERNED_TY_VARS: u32 = 100;
 const NUM_PREINTERNED_FRESH_TYS: u32 = 20;
 const NUM_PREINTERNED_FRESH_INT_TYS: u32 = 3;
 const NUM_PREINTERNED_FRESH_FLOAT_TYS: u32 = 3;
+const NUM_PREINTERNED_ANON_BOUND_TYS_I: u32 = 3;
+const NUM_PREINTERNED_ANON_BOUND_TYS_V: u32 = 20;
 
 // This number may seem high, but it is reached in all but the smallest crates.
 const NUM_PREINTERNED_RE_VARS: u32 = 500;
-const NUM_PREINTERNED_RE_LATE_BOUNDS_I: u32 = 2;
-const NUM_PREINTERNED_RE_LATE_BOUNDS_V: u32 = 20;
+const NUM_PREINTERNED_ANON_RE_BOUNDS_I: u32 = 3;
+const NUM_PREINTERNED_ANON_RE_BOUNDS_V: u32 = 20;
 
 pub struct CommonTypes<'tcx> {
     pub unit: Ty<'tcx>,
@@ -1088,6 +1090,11 @@ pub struct CommonTypes<'tcx> {
 
     /// Pre-interned `Infer(ty::FreshFloatTy(n))` for small values of `n`.
     pub fresh_float_tys: Vec<Ty<'tcx>>,
+
+    /// Pre-interned values of the form:
+    /// `Bound(DebruijnIndex(i), BoundTy { var: v, kind: BoundTyKind::Anon})`
+    /// for small values of `i` and `v`.
+    pub anon_bound_tys: Vec<Vec<Ty<'tcx>>>,
 }
 
 pub struct CommonLifetimes<'tcx> {
@@ -1101,9 +1108,9 @@ pub struct CommonLifetimes<'tcx> {
     pub re_vars: Vec<Region<'tcx>>,
 
     /// Pre-interned values of the form:
-    /// `ReBound(DebruijnIndex(i), BoundRegion { var: v, kind: BrAnon })`
+    /// `ReBound(DebruijnIndex(i), BoundRegion { var: v, kind: BoundRegionKind::Anon })`
     /// for small values of `i` and `v`.
-    pub re_late_bounds: Vec<Vec<Region<'tcx>>>,
+    pub anon_re_bounds: Vec<Vec<Region<'tcx>>>,
 }
 
 pub struct CommonConsts<'tcx> {
@@ -1131,6 +1138,19 @@ impl<'tcx> CommonTypes<'tcx> {
         let fresh_float_tys: Vec<_> =
             (0..NUM_PREINTERNED_FRESH_FLOAT_TYS).map(|n| mk(Infer(ty::FreshFloatTy(n)))).collect();
 
+        let anon_bound_tys = (0..NUM_PREINTERNED_ANON_BOUND_TYS_I)
+            .map(|i| {
+                (0..NUM_PREINTERNED_ANON_BOUND_TYS_V)
+                    .map(|v| {
+                        mk(ty::Bound(
+                            ty::DebruijnIndex::from(i),
+                            ty::BoundTy { var: ty::BoundVar::from(v), kind: ty::BoundTyKind::Anon },
+                        ))
+                    })
+                    .collect()
+            })
+            .collect();
+
         CommonTypes {
             unit: mk(Tuple(List::empty())),
             bool: mk(Bool),
@@ -1161,6 +1181,7 @@ impl<'tcx> CommonTypes<'tcx> {
             fresh_tys,
             fresh_int_tys,
             fresh_float_tys,
+            anon_bound_tys,
         }
     }
 }
@@ -1176,9 +1197,9 @@ impl<'tcx> CommonLifetimes<'tcx> {
         let re_vars =
             (0..NUM_PREINTERNED_RE_VARS).map(|n| mk(ty::ReVar(ty::RegionVid::from(n)))).collect();
 
-        let re_late_bounds = (0..NUM_PREINTERNED_RE_LATE_BOUNDS_I)
+        let anon_re_bounds = (0..NUM_PREINTERNED_ANON_RE_BOUNDS_I)
             .map(|i| {
-                (0..NUM_PREINTERNED_RE_LATE_BOUNDS_V)
+                (0..NUM_PREINTERNED_ANON_RE_BOUNDS_V)
                     .map(|v| {
                         mk(ty::ReBound(
                             ty::DebruijnIndex::from(i),
@@ -1196,7 +1217,7 @@ impl<'tcx> CommonLifetimes<'tcx> {
             re_static: mk(ty::ReStatic),
             re_erased: mk(ty::ReErased),
             re_vars,
-            re_late_bounds,
+            anon_re_bounds,
         }
     }
 }
@@ -1625,7 +1646,11 @@ impl<'tcx> TyCtxt<'tcx> {
 
     /// Allocates a read-only byte or string literal for `mir::interpret` with alignment 1.
     /// Returns the same `AllocId` if called again with the same bytes.
-    pub fn allocate_bytes_dedup(self, bytes: &[u8], salt: usize) -> interpret::AllocId {
+    pub fn allocate_bytes_dedup<'a>(
+        self,
+        bytes: impl Into<Cow<'a, [u8]>>,
+        salt: usize,
+    ) -> interpret::AllocId {
         // Create an allocation that just contains these bytes.
         let alloc = interpret::Allocation::from_bytes_byte_aligned_immutable(bytes, ());
         let alloc = self.mk_const_alloc(alloc);
@@ -3373,6 +3398,11 @@ impl<'tcx> TyCtxt<'tcx> {
         self.resolutions(()).module_children.get(&def_id).map_or(&[], |v| &v[..])
     }
 
+    /// Return the crate imported by given use item.
+    pub fn extern_mod_stmt_cnum(self, def_id: LocalDefId) -> Option<CrateNum> {
+        self.resolutions(()).extern_crate_map.get(&def_id).copied()
+    }
+
     pub fn resolver_for_lowering(self) -> &'tcx Steal<(ty::ResolverAstLowering, Arc<ast::Crate>)> {
         self.resolver_for_lowering_raw(()).0
     }
@@ -3410,6 +3440,20 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn do_not_recommend_impl(self, def_id: DefId) -> bool {
         self.get_diagnostic_attr(def_id, sym::do_not_recommend).is_some()
     }
+
+    /// Whether this def is one of the special bin crate entrypoint functions that must have a
+    /// monomorphization and also not be internalized in the bin crate.
+    pub fn is_entrypoint(self, def_id: DefId) -> bool {
+        if self.is_lang_item(def_id, LangItem::Start) {
+            return true;
+        }
+        if let Some((entry_def_id, _)) = self.entry_fn(())
+            && entry_def_id == def_id
+        {
+            return true;
+        }
+        false
+    }
 }
 
 /// Parameter attributes that can only be determined by examining the body of a function instead
@@ -3428,8 +3472,6 @@ pub struct DeducedParamAttrs {
 }
 
 pub fn provide(providers: &mut Providers) {
-    providers.extern_mod_stmt_cnum =
-        |tcx, id| tcx.resolutions(()).extern_crate_map.get(&id).cloned();
     providers.is_panic_runtime =
         |tcx, LocalCrate| contains_name(tcx.hir_krate_attrs(), sym::panic_runtime);
     providers.is_compiler_builtins =
diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs
index 9ee64df0ad0..fd4472e1f96 100644
--- a/compiler/rustc_middle/src/ty/print/pretty.rs
+++ b/compiler/rustc_middle/src/ty/print/pretty.rs
@@ -86,7 +86,7 @@ macro_rules! define_helper {
 
             impl $helper {
                 pub fn new() -> $helper {
-                    $helper($tl.with(|c| c.replace(true)))
+                    $helper($tl.replace(true))
                 }
             }
 
@@ -100,12 +100,12 @@ macro_rules! define_helper {
 
             impl Drop for $helper {
                 fn drop(&mut self) {
-                    $tl.with(|c| c.set(self.0))
+                    $tl.set(self.0)
                 }
             }
 
             pub fn $name() -> bool {
-                $tl.with(|c| c.get())
+                $tl.get()
             }
         )+
     }
diff --git a/compiler/rustc_middle/src/ty/region.rs b/compiler/rustc_middle/src/ty/region.rs
index 51be93d9a72..5cf96072177 100644
--- a/compiler/rustc_middle/src/ty/region.rs
+++ b/compiler/rustc_middle/src/ty/region.rs
@@ -54,7 +54,7 @@ impl<'tcx> Region<'tcx> {
     ) -> Region<'tcx> {
         // Use a pre-interned one when possible.
         if let ty::BoundRegion { var, kind: ty::BoundRegionKind::Anon } = bound_region
-            && let Some(inner) = tcx.lifetimes.re_late_bounds.get(debruijn.as_usize())
+            && let Some(inner) = tcx.lifetimes.anon_re_bounds.get(debruijn.as_usize())
             && let Some(re) = inner.get(var.as_usize()).copied()
         {
             re
diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs
index ab31d943408..a5fdce93e4b 100644
--- a/compiler/rustc_middle/src/ty/structural_impls.rs
+++ b/compiler/rustc_middle/src/ty/structural_impls.rs
@@ -4,6 +4,7 @@
 //! to help with the tedium.
 
 use std::fmt::{self, Debug};
+use std::marker::PhantomData;
 
 use rustc_abi::TyAndLayout;
 use rustc_hir::def::Namespace;
@@ -234,6 +235,7 @@ TrivialLiftImpls! {
     rustc_abi::ExternAbi,
     rustc_abi::Size,
     rustc_hir::Safety,
+    rustc_middle::mir::ConstValue,
     rustc_type_ir::BoundConstness,
     rustc_type_ir::PredicatePolarity,
     // tidy-alphabetical-end
@@ -250,7 +252,7 @@ TrivialTypeTraversalImpls! {
     crate::mir::BlockTailInfo,
     crate::mir::BorrowKind,
     crate::mir::CastKind,
-    crate::mir::ConstValue<'tcx>,
+    crate::mir::ConstValue,
     crate::mir::CoroutineSavedLocal,
     crate::mir::FakeReadCause,
     crate::mir::Local,
@@ -311,6 +313,13 @@ TrivialTypeTraversalAndLiftImpls! {
 ///////////////////////////////////////////////////////////////////////////
 // Lift implementations
 
+impl<'tcx> Lift<TyCtxt<'tcx>> for PhantomData<&()> {
+    type Lifted = PhantomData<&'tcx ()>;
+    fn lift_to_interner(self, _: TyCtxt<'tcx>) -> Option<Self::Lifted> {
+        Some(PhantomData)
+    }
+}
+
 impl<'tcx, T: Lift<TyCtxt<'tcx>>> Lift<TyCtxt<'tcx>> for Option<T> {
     type Lifted = Option<T::Lifted>;
     fn lift_to_interner(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs
index 8bb3b3f1263..4569596cfbe 100644
--- a/compiler/rustc_middle/src/ty/sty.rs
+++ b/compiler/rustc_middle/src/ty/sty.rs
@@ -485,7 +485,15 @@ impl<'tcx> Ty<'tcx> {
         index: ty::DebruijnIndex,
         bound_ty: ty::BoundTy,
     ) -> Ty<'tcx> {
-        Ty::new(tcx, Bound(index, bound_ty))
+        // Use a pre-interned one when possible.
+        if let ty::BoundTy { var, kind: ty::BoundTyKind::Anon } = bound_ty
+            && let Some(inner) = tcx.types.anon_bound_tys.get(index.as_usize())
+            && let Some(ty) = inner.get(var.as_usize()).copied()
+        {
+            ty
+        } else {
+            Ty::new(tcx, Bound(index, bound_ty))
+        }
     }
 
     #[inline]
diff --git a/compiler/rustc_mir_build/messages.ftl b/compiler/rustc_mir_build/messages.ftl
index e339520cd86..abfe8eb66dd 100644
--- a/compiler/rustc_mir_build/messages.ftl
+++ b/compiler/rustc_mir_build/messages.ftl
@@ -86,10 +86,16 @@ mir_build_confused = missing patterns are not covered because `{$variable}` is i
 
 mir_build_const_continue_bad_const = could not determine the target branch for this `#[const_continue]`
     .label = this value is too generic
-    .note = the value must be a literal or a monomorphic const
 
 mir_build_const_continue_missing_value = a `#[const_continue]` must break to a label with a value
 
+mir_build_const_continue_not_const = could not determine the target branch for this `#[const_continue]`
+    .help = try extracting the expression into a `const` item
+
+mir_build_const_continue_not_const_const_block = `const` blocks may use generics, and are not evaluated early enough
+mir_build_const_continue_not_const_const_other = this value must be a literal or a monomorphic const
+mir_build_const_continue_not_const_constant_parameter = constant parameters may use generics, and are not evaluated early enough
+
 mir_build_const_continue_unknown_jump_target = the target of this `#[const_continue]` is not statically known
     .label = this value must be a literal or a monomorphic const
 
diff --git a/compiler/rustc_mir_build/src/builder/expr/as_constant.rs b/compiler/rustc_mir_build/src/builder/expr/as_constant.rs
index d0d0c21463f..0e0c7a7fa4f 100644
--- a/compiler/rustc_mir_build/src/builder/expr/as_constant.rs
+++ b/compiler/rustc_mir_build/src/builder/expr/as_constant.rs
@@ -3,7 +3,7 @@
 use rustc_abi::Size;
 use rustc_ast as ast;
 use rustc_hir::LangItem;
-use rustc_middle::mir::interpret::{Allocation, CTFE_ALLOC_SALT, LitToConstInput, Scalar};
+use rustc_middle::mir::interpret::{CTFE_ALLOC_SALT, LitToConstInput, Scalar};
 use rustc_middle::mir::*;
 use rustc_middle::thir::*;
 use rustc_middle::ty::{
@@ -120,17 +120,18 @@ fn lit_to_mir_constant<'tcx>(tcx: TyCtxt<'tcx>, lit_input: LitToConstInput<'tcx>
 
     let value = match (lit, lit_ty.kind()) {
         (ast::LitKind::Str(s, _), ty::Ref(_, inner_ty, _)) if inner_ty.is_str() => {
-            let s = s.as_str();
-            let allocation = Allocation::from_bytes_byte_aligned_immutable(s.as_bytes(), ());
-            let allocation = tcx.mk_const_alloc(allocation);
-            ConstValue::Slice { data: allocation, meta: allocation.inner().size().bytes() }
+            let s = s.as_str().as_bytes();
+            let len = s.len();
+            let allocation = tcx.allocate_bytes_dedup(s, CTFE_ALLOC_SALT);
+            ConstValue::Slice { alloc_id: allocation, meta: len.try_into().unwrap() }
         }
-        (ast::LitKind::ByteStr(data, _), ty::Ref(_, inner_ty, _))
+        (ast::LitKind::ByteStr(byte_sym, _), ty::Ref(_, inner_ty, _))
             if matches!(inner_ty.kind(), ty::Slice(_)) =>
         {
-            let allocation = Allocation::from_bytes_byte_aligned_immutable(data.as_byte_str(), ());
-            let allocation = tcx.mk_const_alloc(allocation);
-            ConstValue::Slice { data: allocation, meta: allocation.inner().size().bytes() }
+            let data = byte_sym.as_byte_str();
+            let len = data.len();
+            let allocation = tcx.allocate_bytes_dedup(data, CTFE_ALLOC_SALT);
+            ConstValue::Slice { alloc_id: allocation, meta: len.try_into().unwrap() }
         }
         (ast::LitKind::ByteStr(byte_sym, _), ty::Ref(_, inner_ty, _)) if inner_ty.is_array() => {
             let id = tcx.allocate_bytes_dedup(byte_sym.as_byte_str(), CTFE_ALLOC_SALT);
@@ -138,10 +139,10 @@ fn lit_to_mir_constant<'tcx>(tcx: TyCtxt<'tcx>, lit_input: LitToConstInput<'tcx>
         }
         (ast::LitKind::CStr(byte_sym, _), ty::Ref(_, inner_ty, _)) if matches!(inner_ty.kind(), ty::Adt(def, _) if tcx.is_lang_item(def.did(), LangItem::CStr)) =>
         {
-            let allocation =
-                Allocation::from_bytes_byte_aligned_immutable(byte_sym.as_byte_str(), ());
-            let allocation = tcx.mk_const_alloc(allocation);
-            ConstValue::Slice { data: allocation, meta: allocation.inner().size().bytes() }
+            let data = byte_sym.as_byte_str();
+            let len = data.len();
+            let allocation = tcx.allocate_bytes_dedup(data, CTFE_ALLOC_SALT);
+            ConstValue::Slice { alloc_id: allocation, meta: len.try_into().unwrap() }
         }
         (ast::LitKind::Byte(n), ty::Uint(ty::UintTy::U8)) => {
             ConstValue::Scalar(Scalar::from_uint(n, Size::from_bytes(1)))
diff --git a/compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs b/compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs
index daf8fa5f19e..a4ef6e92739 100644
--- a/compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs
+++ b/compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs
@@ -1,6 +1,6 @@
 //! See docs in `build/expr/mod.rs`.
 
-use rustc_abi::{BackendRepr, FieldIdx, Primitive};
+use rustc_abi::FieldIdx;
 use rustc_hir::lang_items::LangItem;
 use rustc_index::{Idx, IndexVec};
 use rustc_middle::bug;
@@ -9,7 +9,6 @@ use rustc_middle::mir::interpret::Scalar;
 use rustc_middle::mir::*;
 use rustc_middle::thir::*;
 use rustc_middle::ty::cast::{CastTy, mir_cast_kind};
-use rustc_middle::ty::layout::IntegerExt;
 use rustc_middle::ty::util::IntTypeExt;
 use rustc_middle::ty::{self, Ty, UpvarArgs};
 use rustc_span::source_map::Spanned;
@@ -200,8 +199,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 {
                     let discr_ty = adt_def.repr().discr_type().to_ty(this.tcx);
                     let temp = unpack!(block = this.as_temp(block, scope, source, Mutability::Not));
-                    let layout =
-                        this.tcx.layout_of(this.typing_env().as_query_input(source_expr.ty));
                     let discr = this.temp(discr_ty, source_expr.span);
                     this.cfg.push_assign(
                         block,
@@ -209,80 +206,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         discr,
                         Rvalue::Discriminant(temp.into()),
                     );
-                    let (op, ty) = (Operand::Move(discr), discr_ty);
-
-                    if let BackendRepr::Scalar(scalar) = layout.unwrap().backend_repr
-                        && !scalar.is_always_valid(&this.tcx)
-                        && let Primitive::Int(int_width, _signed) = scalar.primitive()
-                    {
-                        let unsigned_ty = int_width.to_ty(this.tcx, false);
-                        let unsigned_place = this.temp(unsigned_ty, expr_span);
-                        this.cfg.push_assign(
-                            block,
-                            source_info,
-                            unsigned_place,
-                            Rvalue::Cast(CastKind::IntToInt, Operand::Copy(discr), unsigned_ty),
-                        );
-
-                        let bool_ty = this.tcx.types.bool;
-                        let range = scalar.valid_range(&this.tcx);
-                        let merge_op =
-                            if range.start <= range.end { BinOp::BitAnd } else { BinOp::BitOr };
-
-                        let mut comparer = |range: u128, bin_op: BinOp| -> Place<'tcx> {
-                            // We can use `ty::TypingEnv::fully_monomorphized()` here
-                            // as we only need it to compute the layout of a primitive.
-                            let range_val = Const::from_bits(
-                                this.tcx,
-                                range,
-                                ty::TypingEnv::fully_monomorphized(),
-                                unsigned_ty,
-                            );
-                            let lit_op = this.literal_operand(expr.span, range_val);
-                            let is_bin_op = this.temp(bool_ty, expr_span);
-                            this.cfg.push_assign(
-                                block,
-                                source_info,
-                                is_bin_op,
-                                Rvalue::BinaryOp(
-                                    bin_op,
-                                    Box::new((Operand::Copy(unsigned_place), lit_op)),
-                                ),
-                            );
-                            is_bin_op
-                        };
-                        let assert_place = if range.start == 0 {
-                            comparer(range.end, BinOp::Le)
-                        } else {
-                            let start_place = comparer(range.start, BinOp::Ge);
-                            let end_place = comparer(range.end, BinOp::Le);
-                            let merge_place = this.temp(bool_ty, expr_span);
-                            this.cfg.push_assign(
-                                block,
-                                source_info,
-                                merge_place,
-                                Rvalue::BinaryOp(
-                                    merge_op,
-                                    Box::new((
-                                        Operand::Move(start_place),
-                                        Operand::Move(end_place),
-                                    )),
-                                ),
-                            );
-                            merge_place
-                        };
-                        this.cfg.push(
-                            block,
-                            Statement::new(
-                                source_info,
-                                StatementKind::Intrinsic(Box::new(NonDivergingIntrinsic::Assume(
-                                    Operand::Move(assert_place),
-                                ))),
-                            ),
-                        );
-                    }
-
-                    (op, ty)
+                    (Operand::Move(discr), discr_ty)
                 } else {
                     let ty = source_expr.ty;
                     let source = unpack!(
diff --git a/compiler/rustc_mir_build/src/builder/mod.rs b/compiler/rustc_mir_build/src/builder/mod.rs
index 3d5f6f4cf45..855cd2f3bc0 100644
--- a/compiler/rustc_mir_build/src/builder/mod.rs
+++ b/compiler/rustc_mir_build/src/builder/mod.rs
@@ -1045,11 +1045,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     }
 }
 
-fn parse_float_into_constval<'tcx>(
-    num: Symbol,
-    float_ty: ty::FloatTy,
-    neg: bool,
-) -> Option<ConstValue<'tcx>> {
+fn parse_float_into_constval(num: Symbol, float_ty: ty::FloatTy, neg: bool) -> Option<ConstValue> {
     parse_float_into_scalar(num, float_ty, neg).map(|s| ConstValue::Scalar(s.into()))
 }
 
diff --git a/compiler/rustc_mir_build/src/builder/scope.rs b/compiler/rustc_mir_build/src/builder/scope.rs
index 12a56d7c5ea..1240b34cf9d 100644
--- a/compiler/rustc_mir_build/src/builder/scope.rs
+++ b/compiler/rustc_mir_build/src/builder/scope.rs
@@ -100,7 +100,9 @@ use tracing::{debug, instrument};
 
 use super::matches::BuiltMatchTree;
 use crate::builder::{BlockAnd, BlockAndExtension, BlockFrame, Builder, CFG};
-use crate::errors::{ConstContinueBadConst, ConstContinueUnknownJumpTarget};
+use crate::errors::{
+    ConstContinueBadConst, ConstContinueNotMonomorphicConst, ConstContinueUnknownJumpTarget,
+};
 
 #[derive(Debug)]
 pub(crate) struct Scopes<'tcx> {
@@ -867,7 +869,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             span_bug!(span, "break value must be a scope")
         };
 
-        let constant = match &self.thir[value].kind {
+        let expr = &self.thir[value];
+        let constant = match &expr.kind {
             ExprKind::Adt(box AdtExpr { variant_index, fields, base, .. }) => {
                 assert!(matches!(base, AdtExprBase::None));
                 assert!(fields.is_empty());
@@ -887,7 +890,27 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     ),
                 }
             }
-            _ => self.as_constant(&self.thir[value]),
+
+            ExprKind::Literal { .. }
+            | ExprKind::NonHirLiteral { .. }
+            | ExprKind::ZstLiteral { .. }
+            | ExprKind::NamedConst { .. } => self.as_constant(&self.thir[value]),
+
+            other => {
+                use crate::errors::ConstContinueNotMonomorphicConstReason as Reason;
+
+                let span = expr.span;
+                let reason = match other {
+                    ExprKind::ConstParam { .. } => Reason::ConstantParameter { span },
+                    ExprKind::ConstBlock { .. } => Reason::ConstBlock { span },
+                    _ => Reason::Other { span },
+                };
+
+                self.tcx
+                    .dcx()
+                    .emit_err(ConstContinueNotMonomorphicConst { span: expr.span, reason });
+                return block.unit();
+            }
         };
 
         let break_index = self
diff --git a/compiler/rustc_mir_build/src/errors.rs b/compiler/rustc_mir_build/src/errors.rs
index 16b49bf384c..f1fbd5c4a49 100644
--- a/compiler/rustc_mir_build/src/errors.rs
+++ b/compiler/rustc_mir_build/src/errors.rs
@@ -1214,6 +1214,38 @@ pub(crate) struct LoopMatchArmWithGuard {
 }
 
 #[derive(Diagnostic)]
+#[diag(mir_build_const_continue_not_const)]
+#[help]
+pub(crate) struct ConstContinueNotMonomorphicConst {
+    #[primary_span]
+    pub span: Span,
+
+    #[subdiagnostic]
+    pub reason: ConstContinueNotMonomorphicConstReason,
+}
+
+#[derive(Subdiagnostic)]
+pub(crate) enum ConstContinueNotMonomorphicConstReason {
+    #[label(mir_build_const_continue_not_const_constant_parameter)]
+    ConstantParameter {
+        #[primary_span]
+        span: Span,
+    },
+
+    #[label(mir_build_const_continue_not_const_const_block)]
+    ConstBlock {
+        #[primary_span]
+        span: Span,
+    },
+
+    #[label(mir_build_const_continue_not_const_const_other)]
+    Other {
+        #[primary_span]
+        span: Span,
+    },
+}
+
+#[derive(Diagnostic)]
 #[diag(mir_build_const_continue_bad_const)]
 pub(crate) struct ConstContinueBadConst {
     #[primary_span]
diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs
index ddeae093df5..ec76076020e 100644
--- a/compiler/rustc_mir_transform/src/coverage/spans.rs
+++ b/compiler/rustc_mir_transform/src/coverage/spans.rs
@@ -1,8 +1,7 @@
 use rustc_data_structures::fx::FxHashSet;
 use rustc_middle::mir;
 use rustc_middle::ty::TyCtxt;
-use rustc_span::source_map::SourceMap;
-use rustc_span::{BytePos, DesugaringKind, ExpnKind, MacroKind, Span};
+use rustc_span::{DesugaringKind, ExpnKind, MacroKind, Span};
 use tracing::instrument;
 
 use crate::coverage::graph::{BasicCoverageBlock, CoverageGraph};
@@ -84,18 +83,8 @@ pub(super) fn extract_refined_covspans<'tcx>(
     // Discard any span that overlaps with a hole.
     discard_spans_overlapping_holes(&mut covspans, &holes);
 
-    // Discard spans that overlap in unwanted ways.
+    // Perform more refinement steps after holes have been dealt with.
     let mut covspans = remove_unwanted_overlapping_spans(covspans);
-
-    // For all empty spans, either enlarge them to be non-empty, or discard them.
-    let source_map = tcx.sess.source_map();
-    covspans.retain_mut(|covspan| {
-        let Some(span) = ensure_non_empty_span(source_map, covspan.span) else { return false };
-        covspan.span = span;
-        true
-    });
-
-    // Merge covspans that can be merged.
     covspans.dedup_by(|b, a| a.merge_if_eligible(b));
 
     code_mappings.extend(covspans.into_iter().map(|Covspan { span, bcb }| {
@@ -241,26 +230,3 @@ fn compare_spans(a: Span, b: Span) -> std::cmp::Ordering {
         // - Both have the same start and span A extends further right
         .then_with(|| Ord::cmp(&a.hi(), &b.hi()).reverse())
 }
-
-fn ensure_non_empty_span(source_map: &SourceMap, span: Span) -> Option<Span> {
-    if !span.is_empty() {
-        return Some(span);
-    }
-
-    // The span is empty, so try to enlarge it to cover an adjacent '{' or '}'.
-    source_map
-        .span_to_source(span, |src, start, end| try {
-            // Adjusting span endpoints by `BytePos(1)` is normally a bug,
-            // but in this case we have specifically checked that the character
-            // we're skipping over is one of two specific ASCII characters, so
-            // adjusting by exactly 1 byte is correct.
-            if src.as_bytes().get(end).copied() == Some(b'{') {
-                Some(span.with_hi(span.hi() + BytePos(1)))
-            } else if start > 0 && src.as_bytes()[start - 1] == b'}' {
-                Some(span.with_lo(span.lo() - BytePos(1)))
-            } else {
-                None
-            }
-        })
-        .ok()?
-}
diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index 6657f89ceb5..dc99b67a1e8 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -1542,7 +1542,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
 fn op_to_prop_const<'tcx>(
     ecx: &mut InterpCx<'tcx, DummyMachine>,
     op: &OpTy<'tcx>,
-) -> Option<ConstValue<'tcx>> {
+) -> Option<ConstValue> {
     // Do not attempt to propagate unsized locals.
     if op.layout.is_unsized() {
         return None;
diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs
index 91c8e64ce9a..1bfd83d97ac 100644
--- a/compiler/rustc_monomorphize/src/collector.rs
+++ b/compiler/rustc_monomorphize/src/collector.rs
@@ -659,10 +659,7 @@ impl<'a, 'tcx> MirUsedCollector<'a, 'tcx> {
     }
 
     /// Evaluates a *not yet monomorphized* constant.
-    fn eval_constant(
-        &mut self,
-        constant: &mir::ConstOperand<'tcx>,
-    ) -> Option<mir::ConstValue<'tcx>> {
+    fn eval_constant(&mut self, constant: &mir::ConstOperand<'tcx>) -> Option<mir::ConstValue> {
         let const_ = self.monomorphize(constant.const_);
         // Evaluate the constant. This makes const eval failure a collection-time error (rather than
         // a codegen-time error). rustc stops after collection if there was an error, so this
@@ -1355,19 +1352,15 @@ fn visit_mentioned_item<'tcx>(
 #[instrument(skip(tcx, output), level = "debug")]
 fn collect_const_value<'tcx>(
     tcx: TyCtxt<'tcx>,
-    value: mir::ConstValue<'tcx>,
+    value: mir::ConstValue,
     output: &mut MonoItems<'tcx>,
 ) {
     match value {
         mir::ConstValue::Scalar(Scalar::Ptr(ptr, _size)) => {
             collect_alloc(tcx, ptr.provenance.alloc_id(), output)
         }
-        mir::ConstValue::Indirect { alloc_id, .. } => collect_alloc(tcx, alloc_id, output),
-        mir::ConstValue::Slice { data, meta: _ } => {
-            for &prov in data.inner().provenance().ptrs().values() {
-                collect_alloc(tcx, prov.alloc_id(), output);
-            }
-        }
+        mir::ConstValue::Indirect { alloc_id, .. }
+        | mir::ConstValue::Slice { alloc_id, meta: _ } => collect_alloc(tcx, alloc_id, output),
         _ => {}
     }
 }
@@ -1582,6 +1575,15 @@ impl<'v> RootCollector<'_, 'v> {
             return;
         };
 
+        let main_instance = Instance::mono(self.tcx, main_def_id);
+        if self.tcx.should_codegen_locally(main_instance) {
+            self.output.push(create_fn_mono_item(
+                self.tcx,
+                main_instance,
+                self.tcx.def_span(main_def_id),
+            ));
+        }
+
         let Some(start_def_id) = self.tcx.lang_items().start_fn() else {
             self.tcx.dcx().emit_fatal(errors::StartNotFound);
         };
diff --git a/compiler/rustc_monomorphize/src/partitioning.rs b/compiler/rustc_monomorphize/src/partitioning.rs
index 69851511fb1..ca8228de57e 100644
--- a/compiler/rustc_monomorphize/src/partitioning.rs
+++ b/compiler/rustc_monomorphize/src/partitioning.rs
@@ -223,11 +223,7 @@ where
         // So even if its mode is LocalCopy, we need to treat it like a root.
         match mono_item.instantiation_mode(cx.tcx) {
             InstantiationMode::GloballyShared { .. } => {}
-            InstantiationMode::LocalCopy => {
-                if !cx.tcx.is_lang_item(mono_item.def_id(), LangItem::Start) {
-                    continue;
-                }
-            }
+            InstantiationMode::LocalCopy => continue,
         }
 
         let characteristic_def_id = characteristic_def_id_of_mono_item(cx.tcx, mono_item);
@@ -821,10 +817,9 @@ fn mono_item_visibility<'tcx>(
         | InstanceKind::FnPtrAddrShim(..) => return Visibility::Hidden,
     };
 
-    // The `start_fn` lang item is actually a monomorphized instance of a
-    // function in the standard library, used for the `main` function. We don't
-    // want to export it so we tag it with `Hidden` visibility but this symbol
-    // is only referenced from the actual `main` symbol which we unfortunately
+    // Both the `start_fn` lang item and `main` itself should not be exported,
+    // so we give them with `Hidden` visibility but these symbols are
+    // only referenced from the actual `main` symbol which we unfortunately
     // don't know anything about during partitioning/collection. As a result we
     // forcibly keep this symbol out of the `internalization_candidates` set.
     //
@@ -834,7 +829,7 @@ fn mono_item_visibility<'tcx>(
     //        from the `main` symbol we'll generate later.
     //
     //        This may be fixable with a new `InstanceKind` perhaps? Unsure!
-    if tcx.is_lang_item(def_id, LangItem::Start) {
+    if tcx.is_entrypoint(def_id) {
         *can_be_internalized = false;
         return Visibility::Hidden;
     }
diff --git a/compiler/rustc_next_trait_solver/src/solve/assembly/mod.rs b/compiler/rustc_next_trait_solver/src/solve/assembly/mod.rs
index c281ef216c7..b2d40146348 100644
--- a/compiler/rustc_next_trait_solver/src/solve/assembly/mod.rs
+++ b/compiler/rustc_next_trait_solver/src/solve/assembly/mod.rs
@@ -10,8 +10,8 @@ use rustc_type_ir::inherent::*;
 use rustc_type_ir::lang_items::TraitSolverLangItem;
 use rustc_type_ir::solve::SizedTraitKind;
 use rustc_type_ir::{
-    self as ty, Interner, TypeFoldable, TypeSuperVisitable, TypeVisitable, TypeVisitableExt as _,
-    TypeVisitor, TypingMode, Upcast as _, elaborate,
+    self as ty, Interner, TypeFlags, TypeFoldable, TypeSuperVisitable, TypeVisitable,
+    TypeVisitableExt as _, TypeVisitor, TypingMode, Upcast as _, elaborate,
 };
 use tracing::{debug, instrument};
 
@@ -132,6 +132,7 @@ where
         })
         .enter(|ecx| {
             Self::match_assumption(ecx, goal, assumption, |ecx| {
+                ecx.try_evaluate_added_goals()?;
                 source.set(ecx.characterize_param_env_assumption(goal.param_env, assumption)?);
                 ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
             })
@@ -1069,8 +1070,10 @@ where
             } else {
                 ControlFlow::Continue(())
             }
-        } else {
+        } else if ty.has_type_flags(TypeFlags::HAS_PLACEHOLDER | TypeFlags::HAS_RE_INFER) {
             ty.super_visit_with(self)
+        } else {
+            ControlFlow::Continue(())
         }
     }
 
@@ -1086,8 +1089,10 @@ where
             } else {
                 ControlFlow::Continue(())
             }
-        } else {
+        } else if ct.has_type_flags(TypeFlags::HAS_PLACEHOLDER | TypeFlags::HAS_RE_INFER) {
             ct.super_visit_with(self)
+        } else {
+            ControlFlow::Continue(())
         }
     }
 
diff --git a/compiler/rustc_parse/Cargo.toml b/compiler/rustc_parse/Cargo.toml
index a92012f8329..88f93782de1 100644
--- a/compiler/rustc_parse/Cargo.toml
+++ b/compiler/rustc_parse/Cargo.toml
@@ -9,6 +9,7 @@ bitflags = "2.4.1"
 rustc-literal-escaper = "0.0.5"
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
+rustc_attr_parsing = { path = "../rustc_attr_parsing" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_feature = { path = "../rustc_feature" }
diff --git a/compiler/rustc_parse/src/validate_attr.rs b/compiler/rustc_parse/src/validate_attr.rs
index ed7fb774908..bc4c605afad 100644
--- a/compiler/rustc_parse/src/validate_attr.rs
+++ b/compiler/rustc_parse/src/validate_attr.rs
@@ -1,11 +1,14 @@
 //! Meta-syntax validation logic of attributes for post-expansion.
 
+use std::slice;
+
 use rustc_ast::token::Delimiter;
 use rustc_ast::tokenstream::DelimSpan;
 use rustc_ast::{
     self as ast, AttrArgs, Attribute, DelimArgs, MetaItem, MetaItemInner, MetaItemKind, NodeId,
     Path, Safety,
 };
+use rustc_attr_parsing::{AttributeParser, Late};
 use rustc_errors::{Applicability, DiagCtxtHandle, FatalError, PResult};
 use rustc_feature::{AttributeSafety, AttributeTemplate, BUILTIN_ATTRIBUTE_MAP, BuiltinAttribute};
 use rustc_session::errors::report_lit_error;
@@ -266,64 +269,7 @@ pub fn check_builtin_meta_item(
 ) {
     if !is_attr_template_compatible(&template, &meta.kind) {
         // attrs with new parsers are locally validated so excluded here
-        if matches!(
-            name,
-            sym::inline
-                | sym::export_stable
-                | sym::ffi_const
-                | sym::ffi_pure
-                | sym::rustc_std_internal_symbol
-                | sym::may_dangle
-                | sym::rustc_as_ptr
-                | sym::rustc_pub_transparent
-                | sym::rustc_const_stable_indirect
-                | sym::rustc_force_inline
-                | sym::rustc_confusables
-                | sym::rustc_skip_during_method_dispatch
-                | sym::rustc_pass_by_value
-                | sym::rustc_deny_explicit_impl
-                | sym::rustc_do_not_implement_via_object
-                | sym::rustc_coinductive
-                | sym::const_trait
-                | sym::rustc_specialization_trait
-                | sym::rustc_unsafe_specialization_marker
-                | sym::rustc_allow_incoherent_impl
-                | sym::rustc_coherence_is_core
-                | sym::marker
-                | sym::fundamental
-                | sym::rustc_paren_sugar
-                | sym::type_const
-                | sym::repr
-                // FIXME(#82232, #143834): temporarily renamed to mitigate `#[align]` nameres
-                // ambiguity
-                | sym::rustc_align
-                | sym::deprecated
-                | sym::optimize
-                | sym::pointee
-                | sym::cold
-                | sym::target_feature
-                | sym::rustc_allow_const_fn_unstable
-                | sym::macro_use
-                | sym::macro_escape
-                | sym::naked
-                | sym::no_mangle
-                | sym::non_exhaustive
-                | sym::omit_gdb_pretty_printer_section
-                | sym::path
-                | sym::ignore
-                | sym::must_use
-                | sym::track_caller
-                | sym::link_name
-                | sym::link_ordinal
-                | sym::export_name
-                | sym::rustc_macro_transparency
-                | sym::link_section
-                | sym::rustc_layout_scalar_valid_range_start
-                | sym::rustc_layout_scalar_valid_range_end
-                | sym::no_implicit_prelude
-                | sym::automatically_derived
-                | sym::coverage
-        ) {
+        if AttributeParser::<Late>::is_parsed_attribute(slice::from_ref(&name)) {
             return;
         }
         emit_malformed_attribute(psess, style, meta.span, name, template);
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index 04c1933f243..4b524bb2bd2 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -1258,7 +1258,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
             return;
         }
 
-        if self.tcx.extern_mod_stmt_cnum(hir_id.owner).is_none() {
+        if self.tcx.extern_mod_stmt_cnum(hir_id.owner.def_id).is_none() {
             self.tcx.emit_node_span_lint(
                 INVALID_DOC_ATTRIBUTES,
                 hir_id,
diff --git a/compiler/rustc_pattern_analysis/src/constructor.rs b/compiler/rustc_pattern_analysis/src/constructor.rs
index 09685640e50..9a9e0db964c 100644
--- a/compiler/rustc_pattern_analysis/src/constructor.rs
+++ b/compiler/rustc_pattern_analysis/src/constructor.rs
@@ -950,9 +950,7 @@ impl<Cx: PatCx> Constructor<Cx> {
                 }
             }
             Never => write!(f, "!")?,
-            Wildcard | Missing | NonExhaustive | Hidden | PrivateUninhabited => {
-                write!(f, "_ : {:?}", ty)?
-            }
+            Wildcard | Missing | NonExhaustive | Hidden | PrivateUninhabited => write!(f, "_")?,
         }
         Ok(())
     }
diff --git a/compiler/rustc_pattern_analysis/src/lib.rs b/compiler/rustc_pattern_analysis/src/lib.rs
index 66df35f9ee4..d9bb93a829b 100644
--- a/compiler/rustc_pattern_analysis/src/lib.rs
+++ b/compiler/rustc_pattern_analysis/src/lib.rs
@@ -57,6 +57,13 @@ pub trait PatCx: Sized + fmt::Debug {
 
     fn is_exhaustive_patterns_feature_on(&self) -> bool;
 
+    /// Whether to ensure the non-exhaustiveness witnesses we report for a complete set. This is
+    /// `false` by default to avoid some exponential blowup cases such as
+    /// <https://github.com/rust-lang/rust/issues/118437>.
+    fn exhaustive_witnesses(&self) -> bool {
+        false
+    }
+
     /// The number of fields for this constructor.
     fn ctor_arity(&self, ctor: &Constructor<Self>, ty: &Self::Ty) -> usize;
 
diff --git a/compiler/rustc_pattern_analysis/src/usefulness.rs b/compiler/rustc_pattern_analysis/src/usefulness.rs
index b1c646e9884..19446a1efe9 100644
--- a/compiler/rustc_pattern_analysis/src/usefulness.rs
+++ b/compiler/rustc_pattern_analysis/src/usefulness.rs
@@ -994,7 +994,8 @@ impl<Cx: PatCx> PlaceInfo<Cx> {
         if !missing_ctors.is_empty() && !report_individual_missing_ctors {
             // Report `_` as missing.
             missing_ctors = vec![Constructor::Wildcard];
-        } else if missing_ctors.iter().any(|c| c.is_non_exhaustive()) {
+        } else if missing_ctors.iter().any(|c| c.is_non_exhaustive()) && !cx.exhaustive_witnesses()
+        {
             // We need to report a `_` anyway, so listing other constructors would be redundant.
             // `NonExhaustive` is displayed as `_` just like `Wildcard`, but it will be picked
             // up by diagnostics to add a note about why `_` is required here.
@@ -1747,7 +1748,9 @@ fn compute_exhaustiveness_and_usefulness<'a, 'p, Cx: PatCx>(
         // `ctor` is *irrelevant* if there's another constructor in `split_ctors` that matches
         // strictly fewer rows. In that case we can sometimes skip it. See the top of the file for
         // details.
-        let ctor_is_relevant = matches!(ctor, Constructor::Missing) || missing_ctors.is_empty();
+        let ctor_is_relevant = matches!(ctor, Constructor::Missing)
+            || missing_ctors.is_empty()
+            || mcx.tycx.exhaustive_witnesses();
         let mut spec_matrix = matrix.specialize_constructor(pcx, &ctor, ctor_is_relevant)?;
         let mut witnesses = ensure_sufficient_stack(|| {
             compute_exhaustiveness_and_usefulness(mcx, &mut spec_matrix)
diff --git a/compiler/rustc_pattern_analysis/tests/common/mod.rs b/compiler/rustc_pattern_analysis/tests/common/mod.rs
index 0b939ef7816..94f2a127b9b 100644
--- a/compiler/rustc_pattern_analysis/tests/common/mod.rs
+++ b/compiler/rustc_pattern_analysis/tests/common/mod.rs
@@ -1,3 +1,4 @@
+#![allow(dead_code, unreachable_pub)]
 use rustc_pattern_analysis::constructor::{
     Constructor, ConstructorSet, IntRange, MaybeInfiniteInt, RangeEnd, VariantVisibility,
 };
@@ -22,8 +23,10 @@ fn init_tracing() {
         .try_init();
 }
 
+pub(super) const UNIT: Ty = Ty::Tuple(&[]);
+pub(super) const NEVER: Ty = Ty::Enum(&[]);
+
 /// A simple set of types.
-#[allow(dead_code)]
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
 pub(super) enum Ty {
     /// Booleans
@@ -38,6 +41,8 @@ pub(super) enum Ty {
     BigStruct { arity: usize, ty: &'static Ty },
     /// A enum with `arity` variants of type `ty`.
     BigEnum { arity: usize, ty: &'static Ty },
+    /// Like `Enum` but non-exhaustive.
+    NonExhaustiveEnum(&'static [Ty]),
 }
 
 /// The important logic.
@@ -47,7 +52,7 @@ impl Ty {
         match (ctor, *self) {
             (Struct, Ty::Tuple(tys)) => tys.iter().copied().collect(),
             (Struct, Ty::BigStruct { arity, ty }) => (0..arity).map(|_| *ty).collect(),
-            (Variant(i), Ty::Enum(tys)) => vec![tys[*i]],
+            (Variant(i), Ty::Enum(tys) | Ty::NonExhaustiveEnum(tys)) => vec![tys[*i]],
             (Variant(_), Ty::BigEnum { ty, .. }) => vec![*ty],
             (Bool(..) | IntRange(..) | NonExhaustive | Missing | Wildcard, _) => vec![],
             _ => panic!("Unexpected ctor {ctor:?} for type {self:?}"),
@@ -61,6 +66,7 @@ impl Ty {
             Ty::Enum(tys) => tys.iter().all(|ty| ty.is_empty()),
             Ty::BigStruct { arity, ty } => arity != 0 && ty.is_empty(),
             Ty::BigEnum { arity, ty } => arity == 0 || ty.is_empty(),
+            Ty::NonExhaustiveEnum(..) => false,
         }
     }
 
@@ -90,6 +96,19 @@ impl Ty {
                     .collect(),
                 non_exhaustive: false,
             },
+            Ty::NonExhaustiveEnum(tys) => ConstructorSet::Variants {
+                variants: tys
+                    .iter()
+                    .map(|ty| {
+                        if ty.is_empty() {
+                            VariantVisibility::Empty
+                        } else {
+                            VariantVisibility::Visible
+                        }
+                    })
+                    .collect(),
+                non_exhaustive: true,
+            },
             Ty::BigEnum { arity: 0, .. } => ConstructorSet::NoConstructors,
             Ty::BigEnum { arity, ty } => {
                 let vis = if ty.is_empty() {
@@ -113,7 +132,9 @@ impl Ty {
         match (*self, ctor) {
             (Ty::Tuple(..), _) => Ok(()),
             (Ty::BigStruct { .. }, _) => write!(f, "BigStruct"),
-            (Ty::Enum(..), Constructor::Variant(i)) => write!(f, "Enum::Variant{i}"),
+            (Ty::Enum(..) | Ty::NonExhaustiveEnum(..), Constructor::Variant(i)) => {
+                write!(f, "Enum::Variant{i}")
+            }
             (Ty::BigEnum { .. }, Constructor::Variant(i)) => write!(f, "BigEnum::Variant{i}"),
             _ => write!(f, "{:?}::{:?}", self, ctor),
         }
@@ -126,10 +147,11 @@ pub(super) fn compute_match_usefulness<'p>(
     ty: Ty,
     scrut_validity: PlaceValidity,
     complexity_limit: usize,
+    exhaustive_witnesses: bool,
 ) -> Result<UsefulnessReport<'p, Cx>, ()> {
     init_tracing();
     rustc_pattern_analysis::usefulness::compute_match_usefulness(
-        &Cx,
+        &Cx { exhaustive_witnesses },
         arms,
         ty,
         scrut_validity,
@@ -138,7 +160,9 @@ pub(super) fn compute_match_usefulness<'p>(
 }
 
 #[derive(Debug)]
-pub(super) struct Cx;
+pub(super) struct Cx {
+    exhaustive_witnesses: bool,
+}
 
 /// The context for pattern analysis. Forwards anything interesting to `Ty` methods.
 impl PatCx for Cx {
@@ -153,6 +177,10 @@ impl PatCx for Cx {
         false
     }
 
+    fn exhaustive_witnesses(&self) -> bool {
+        self.exhaustive_witnesses
+    }
+
     fn ctor_arity(&self, ctor: &Constructor<Self>, ty: &Self::Ty) -> usize {
         ty.sub_tys(ctor).len()
     }
@@ -219,16 +247,18 @@ macro_rules! pats {
     // Entrypoint
     // Parse `type; ..`
     ($ty:expr; $($rest:tt)*) => {{
-        #[allow(unused_imports)]
+        #[allow(unused)]
         use rustc_pattern_analysis::{
             constructor::{Constructor, IntRange, MaybeInfiniteInt, RangeEnd},
-            pat::DeconstructedPat,
+            pat::{DeconstructedPat, IndexedPat},
         };
         let ty = $ty;
         // The heart of the macro is designed to push `IndexedPat`s into a `Vec`, so we work around
         // that.
+        #[allow(unused)]
         let sub_tys = ::std::iter::repeat(&ty);
-        let mut vec = Vec::new();
+        #[allow(unused)]
+        let mut vec: Vec<IndexedPat<_>> = Vec::new();
         pats!(@ctor(vec:vec, sub_tys:sub_tys, idx:0) $($rest)*);
         vec.into_iter().map(|ipat| ipat.pat).collect::<Vec<_>>()
     }};
@@ -263,6 +293,8 @@ macro_rules! pats {
         let ctor = Constructor::Wildcard;
         pats!(@pat($($args)*, ctor:ctor) $($rest)*)
     }};
+    // Nothing
+    (@ctor($($args:tt)*)) => {};
 
     // Integers and int ranges
     (@ctor($($args:tt)*) $($start:literal)?..$end:literal $($rest:tt)*) => {{
diff --git a/compiler/rustc_pattern_analysis/tests/complexity.rs b/compiler/rustc_pattern_analysis/tests/complexity.rs
index 93aecafe48d..4754476f383 100644
--- a/compiler/rustc_pattern_analysis/tests/complexity.rs
+++ b/compiler/rustc_pattern_analysis/tests/complexity.rs
@@ -16,7 +16,7 @@ fn check(patterns: &[DeconstructedPat<Cx>], complexity_limit: usize) -> Result<(
     let ty = *patterns[0].ty();
     let arms: Vec<_> =
         patterns.iter().map(|pat| MatchArm { pat, has_guard: false, arm_data: () }).collect();
-    compute_match_usefulness(arms.as_slice(), ty, PlaceValidity::ValidOnly, complexity_limit)
+    compute_match_usefulness(arms.as_slice(), ty, PlaceValidity::ValidOnly, complexity_limit, false)
         .map(|_report| ())
 }
 
diff --git a/compiler/rustc_pattern_analysis/tests/exhaustiveness.rs b/compiler/rustc_pattern_analysis/tests/exhaustiveness.rs
index 3b8f346ef19..14ca0d057f0 100644
--- a/compiler/rustc_pattern_analysis/tests/exhaustiveness.rs
+++ b/compiler/rustc_pattern_analysis/tests/exhaustiveness.rs
@@ -11,16 +11,30 @@ use rustc_pattern_analysis::usefulness::PlaceValidity;
 mod common;
 
 /// Analyze a match made of these patterns.
-fn check(patterns: Vec<DeconstructedPat<Cx>>) -> Vec<WitnessPat<Cx>> {
-    let ty = *patterns[0].ty();
+fn run(
+    ty: Ty,
+    patterns: Vec<DeconstructedPat<Cx>>,
+    exhaustive_witnesses: bool,
+) -> Vec<WitnessPat<Cx>> {
     let arms: Vec<_> =
         patterns.iter().map(|pat| MatchArm { pat, has_guard: false, arm_data: () }).collect();
-    let report =
-        compute_match_usefulness(arms.as_slice(), ty, PlaceValidity::ValidOnly, usize::MAX)
-            .unwrap();
+    let report = compute_match_usefulness(
+        arms.as_slice(),
+        ty,
+        PlaceValidity::ValidOnly,
+        usize::MAX,
+        exhaustive_witnesses,
+    )
+    .unwrap();
     report.non_exhaustiveness_witnesses
 }
 
+/// Analyze a match made of these patterns. Panics if there are no patterns
+fn check(patterns: Vec<DeconstructedPat<Cx>>) -> Vec<WitnessPat<Cx>> {
+    let ty = *patterns[0].ty();
+    run(ty, patterns, true)
+}
+
 #[track_caller]
 fn assert_exhaustive(patterns: Vec<DeconstructedPat<Cx>>) {
     let witnesses = check(patterns);
@@ -35,6 +49,26 @@ fn assert_non_exhaustive(patterns: Vec<DeconstructedPat<Cx>>) {
     assert!(!witnesses.is_empty())
 }
 
+use WhichWitnesses::*;
+enum WhichWitnesses {
+    AllOfThem,
+    OnlySome,
+}
+
+#[track_caller]
+/// We take the type as input to support empty matches.
+fn assert_witnesses(
+    which: WhichWitnesses,
+    ty: Ty,
+    patterns: Vec<DeconstructedPat<Cx>>,
+    expected: Vec<&str>,
+) {
+    let exhaustive_wit = matches!(which, AllOfThem);
+    let witnesses = run(ty, patterns, exhaustive_wit);
+    let witnesses: Vec<_> = witnesses.iter().map(|w| format!("{w:?}")).collect();
+    assert_eq!(witnesses, expected)
+}
+
 #[test]
 fn test_int_ranges() {
     let ty = Ty::U8;
@@ -59,6 +93,8 @@ fn test_int_ranges() {
 
 #[test]
 fn test_nested() {
+    // enum E { A(bool), B(bool) }
+    // ty = (E, E)
     let ty = Ty::BigStruct { arity: 2, ty: &Ty::BigEnum { arity: 2, ty: &Ty::Bool } };
     assert_non_exhaustive(pats!(ty;
         Struct(Variant.0, _),
@@ -79,9 +115,73 @@ fn test_nested() {
 }
 
 #[test]
+fn test_witnesses() {
+    // TY = Option<bool>
+    const TY: Ty = Ty::Enum(&[Ty::Bool, UNIT]);
+    // ty = (Option<bool>, Option<bool>)
+    let ty = Ty::Tuple(&[TY, TY]);
+    assert_witnesses(AllOfThem, ty, vec![], vec!["(_, _)"]);
+    assert_witnesses(
+        OnlySome,
+        ty,
+        pats!(ty;
+            (Variant.0(false), Variant.0(false)),
+        ),
+        vec!["(Enum::Variant1(_), _)"],
+    );
+    assert_witnesses(
+        AllOfThem,
+        ty,
+        pats!(ty;
+            (Variant.0(false), Variant.0(false)),
+        ),
+        vec![
+            "(Enum::Variant0(false), Enum::Variant0(true))",
+            "(Enum::Variant0(false), Enum::Variant1(_))",
+            "(Enum::Variant0(true), _)",
+            "(Enum::Variant1(_), _)",
+        ],
+    );
+    assert_witnesses(
+        OnlySome,
+        ty,
+        pats!(ty;
+            (_, Variant.0(false)),
+        ),
+        vec!["(_, Enum::Variant1(_))"],
+    );
+    assert_witnesses(
+        AllOfThem,
+        ty,
+        pats!(ty;
+            (_, Variant.0(false)),
+        ),
+        vec!["(_, Enum::Variant0(true))", "(_, Enum::Variant1(_))"],
+    );
+
+    let ty = Ty::NonExhaustiveEnum(&[UNIT, UNIT, UNIT]);
+    assert_witnesses(
+        OnlySome,
+        ty,
+        pats!(ty;
+            Variant.0,
+        ),
+        vec!["_"],
+    );
+    assert_witnesses(
+        AllOfThem,
+        ty,
+        pats!(ty;
+            Variant.0,
+        ),
+        vec!["Enum::Variant1(_)", "Enum::Variant2(_)", "_"],
+    );
+}
+
+#[test]
 fn test_empty() {
     // `TY = Result<bool, !>`
-    const TY: Ty = Ty::Enum(&[Ty::Bool, Ty::Enum(&[])]);
+    const TY: Ty = Ty::Enum(&[Ty::Bool, NEVER]);
     assert_exhaustive(pats!(TY;
         Variant.0,
     ));
diff --git a/compiler/rustc_pattern_analysis/tests/intersection.rs b/compiler/rustc_pattern_analysis/tests/intersection.rs
index 8e6f84dcbc8..d4d390517e2 100644
--- a/compiler/rustc_pattern_analysis/tests/intersection.rs
+++ b/compiler/rustc_pattern_analysis/tests/intersection.rs
@@ -16,7 +16,7 @@ fn check(patterns: Vec<DeconstructedPat<Cx>>) -> Vec<Vec<usize>> {
     let arms: Vec<_> =
         patterns.iter().map(|pat| MatchArm { pat, has_guard: false, arm_data: () }).collect();
     let report =
-        compute_match_usefulness(arms.as_slice(), ty, PlaceValidity::ValidOnly, usize::MAX)
+        compute_match_usefulness(arms.as_slice(), ty, PlaceValidity::ValidOnly, usize::MAX, false)
             .unwrap();
     report.arm_intersections.into_iter().map(|bitset| bitset.iter().collect()).collect()
 }
diff --git a/compiler/rustc_privacy/src/lib.rs b/compiler/rustc_privacy/src/lib.rs
index 9dd80bc9964..6fd2b7fc12f 100644
--- a/compiler/rustc_privacy/src/lib.rs
+++ b/compiler/rustc_privacy/src/lib.rs
@@ -26,7 +26,7 @@ use rustc_errors::{MultiSpan, listify};
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::{CRATE_DEF_ID, DefId, LocalDefId, LocalModDefId};
 use rustc_hir::intravisit::{self, InferKind, Visitor};
-use rustc_hir::{AmbigArg, ForeignItemKind, ItemId, ItemKind, PatKind};
+use rustc_hir::{AmbigArg, ForeignItemId, ItemId, PatKind};
 use rustc_middle::middle::privacy::{EffectiveVisibilities, EffectiveVisibility, Level};
 use rustc_middle::query::Providers;
 use rustc_middle::ty::print::PrintTraitRefExt as _;
@@ -599,18 +599,13 @@ impl<'tcx> EmbargoVisitor<'tcx> {
 
             DefKind::Struct | DefKind::Union => {
                 // While structs and unions have type privacy, their fields do not.
-                let item = self.tcx.hir_expect_item(def_id);
-                if let hir::ItemKind::Struct(_, _, ref struct_def)
-                | hir::ItemKind::Union(_, _, ref struct_def) = item.kind
-                {
-                    for field in struct_def.fields() {
-                        let field_vis = self.tcx.local_visibility(field.def_id);
-                        if field_vis.is_accessible_from(module, self.tcx) {
-                            self.reach(field.def_id, macro_ev).ty();
-                        }
+                let struct_def = self.tcx.adt_def(def_id);
+                for field in struct_def.non_enum_variant().fields.iter() {
+                    let def_id = field.did.expect_local();
+                    let field_vis = self.tcx.local_visibility(def_id);
+                    if field_vis.is_accessible_from(module, self.tcx) {
+                        self.reach(def_id, macro_ev).ty();
                     }
-                } else {
-                    bug!("item {:?} with DefKind {:?}", item, def_kind);
                 }
             }
 
@@ -1644,66 +1639,29 @@ impl<'tcx> PrivateItemsInPublicInterfacesChecker<'_, 'tcx> {
                 self.check(def_id, item_visibility, effective_vis).generics().predicates();
             }
             DefKind::Enum => {
-                let item = tcx.hir_item(id);
-                if let hir::ItemKind::Enum(_, _, ref def) = item.kind {
-                    self.check_unnameable(item.owner_id.def_id, effective_vis);
-
-                    self.check(item.owner_id.def_id, item_visibility, effective_vis)
-                        .generics()
-                        .predicates();
-
-                    for variant in def.variants {
-                        for field in variant.data.fields() {
-                            self.check(field.def_id, item_visibility, effective_vis).ty();
-                        }
-                    }
-                }
-            }
-            // Subitems of foreign modules have their own publicity.
-            DefKind::ForeignMod => {
-                let item = tcx.hir_item(id);
-                if let hir::ItemKind::ForeignMod { items, .. } = item.kind {
-                    for &foreign_item in items {
-                        let foreign_item = tcx.hir_foreign_item(foreign_item);
-
-                        let ev = self.get(foreign_item.owner_id.def_id);
-                        let vis = tcx.local_visibility(foreign_item.owner_id.def_id);
-
-                        if let ForeignItemKind::Type = foreign_item.kind {
-                            self.check_unnameable(foreign_item.owner_id.def_id, ev);
-                        }
+                self.check_unnameable(def_id, effective_vis);
+                self.check(def_id, item_visibility, effective_vis).generics().predicates();
 
-                        self.check(foreign_item.owner_id.def_id, vis, ev)
-                            .generics()
-                            .predicates()
-                            .ty();
-                    }
+                let adt = tcx.adt_def(id.owner_id);
+                for field in adt.all_fields() {
+                    self.check(field.did.expect_local(), item_visibility, effective_vis).ty();
                 }
             }
             // Subitems of structs and unions have their own publicity.
             DefKind::Struct | DefKind::Union => {
-                let item = tcx.hir_item(id);
-                if let hir::ItemKind::Struct(_, _, ref struct_def)
-                | hir::ItemKind::Union(_, _, ref struct_def) = item.kind
-                {
-                    self.check_unnameable(item.owner_id.def_id, effective_vis);
-                    self.check(item.owner_id.def_id, item_visibility, effective_vis)
-                        .generics()
-                        .predicates();
+                self.check_unnameable(def_id, effective_vis);
+                self.check(def_id, item_visibility, effective_vis).generics().predicates();
 
-                    for field in struct_def.fields() {
-                        let field_visibility = tcx.local_visibility(field.def_id);
-                        let field_ev = self.get(field.def_id);
+                let adt = tcx.adt_def(id.owner_id);
+                for field in adt.all_fields() {
+                    let visibility = min(item_visibility, field.vis.expect_local(), tcx);
+                    let field_ev = self.get(field.did.expect_local());
 
-                        self.check(
-                            field.def_id,
-                            min(item_visibility, field_visibility, tcx),
-                            field_ev,
-                        )
-                        .ty();
-                    }
+                    self.check(field.did.expect_local(), visibility, field_ev).ty();
                 }
             }
+            // Subitems of foreign modules have their own publicity.
+            DefKind::ForeignMod => {}
             // An inherent impl is public when its type is public
             // Subitems of inherent impls have their own publicity.
             // A trait impl is public when both its type and its trait are public
@@ -1763,6 +1721,19 @@ impl<'tcx> PrivateItemsInPublicInterfacesChecker<'_, 'tcx> {
             _ => {}
         }
     }
+
+    fn check_foreign_item(&mut self, id: ForeignItemId) {
+        let tcx = self.tcx;
+        let def_id = id.owner_id.def_id;
+        let item_visibility = tcx.local_visibility(def_id);
+        let effective_vis = self.get(def_id);
+
+        if let DefKind::ForeignTy = self.tcx.def_kind(def_id) {
+            self.check_unnameable(def_id, effective_vis);
+        }
+
+        self.check(def_id, item_visibility, effective_vis).generics().predicates().ty();
+    }
 }
 
 pub fn provide(providers: &mut Providers) {
@@ -1791,20 +1762,13 @@ fn check_mod_privacy(tcx: TyCtxt<'_>, module_def_id: LocalModDefId) {
         if let Some(body_id) = tcx.hir_maybe_body_owned_by(def_id) {
             visitor.visit_nested_body(body_id.id());
         }
-    }
 
-    for id in module.free_items() {
-        if let ItemKind::Impl(i) = tcx.hir_item(id).kind {
-            if let Some(item) = i.of_trait {
-                let trait_ref = tcx.impl_trait_ref(id.owner_id.def_id).unwrap();
-                let trait_ref = trait_ref.instantiate_identity();
-                visitor.span = item.path.span;
-                let _ = visitor.visit_def_id(
-                    trait_ref.def_id,
-                    "trait",
-                    &trait_ref.print_only_trait_path(),
-                );
-            }
+        if let DefKind::Impl { of_trait: true } = tcx.def_kind(def_id) {
+            let trait_ref = tcx.impl_trait_ref(def_id).unwrap();
+            let trait_ref = trait_ref.instantiate_identity();
+            visitor.span = tcx.hir_expect_item(def_id).expect_impl().of_trait.unwrap().path.span;
+            let _ =
+                visitor.visit_def_id(trait_ref.def_id, "trait", &trait_ref.print_only_trait_path());
         }
     }
 }
@@ -1895,7 +1859,11 @@ fn check_private_in_public(tcx: TyCtxt<'_>, (): ()) {
     // Check for private types in public interfaces.
     let mut checker = PrivateItemsInPublicInterfacesChecker { tcx, effective_visibilities };
 
-    for id in tcx.hir_free_items() {
+    let crate_items = tcx.hir_crate_items(());
+    for id in crate_items.free_items() {
         checker.check_item(id);
     }
+    for id in crate_items.foreign_items() {
+        checker.check_foreign_item(id);
+    }
 }
diff --git a/compiler/rustc_public/src/alloc.rs b/compiler/rustc_public/src/alloc.rs
index 75ad31022ff..0c35b3b25df 100644
--- a/compiler/rustc_public/src/alloc.rs
+++ b/compiler/rustc_public/src/alloc.rs
@@ -33,7 +33,7 @@ fn new_empty_allocation(align: Align) -> Allocation {
 #[allow(rustc::usage_of_qualified_ty)]
 pub(crate) fn new_allocation<'tcx>(
     ty: rustc_middle::ty::Ty<'tcx>,
-    const_value: ConstValue<'tcx>,
+    const_value: ConstValue,
     tables: &mut Tables<'tcx, BridgeTys>,
     cx: &CompilerCtxt<'tcx, BridgeTys>,
 ) -> Allocation {
@@ -44,7 +44,7 @@ pub(crate) fn new_allocation<'tcx>(
 #[allow(rustc::usage_of_qualified_ty)]
 pub(crate) fn try_new_allocation<'tcx>(
     ty: rustc_middle::ty::Ty<'tcx>,
-    const_value: ConstValue<'tcx>,
+    const_value: ConstValue,
     tables: &mut Tables<'tcx, BridgeTys>,
     cx: &CompilerCtxt<'tcx, BridgeTys>,
 ) -> Result<Allocation, Error> {
@@ -54,8 +54,8 @@ pub(crate) fn try_new_allocation<'tcx>(
             alloc::try_new_scalar(layout, scalar, cx).map(|alloc| alloc.stable(tables, cx))
         }
         ConstValue::ZeroSized => Ok(new_empty_allocation(layout.align.abi)),
-        ConstValue::Slice { data, meta } => {
-            alloc::try_new_slice(layout, data, meta, cx).map(|alloc| alloc.stable(tables, cx))
+        ConstValue::Slice { alloc_id, meta } => {
+            alloc::try_new_slice(layout, alloc_id, meta, cx).map(|alloc| alloc.stable(tables, cx))
         }
         ConstValue::Indirect { alloc_id, offset } => {
             let alloc = alloc::try_new_indirect(alloc_id, cx);
diff --git a/compiler/rustc_public/src/mir/body.rs b/compiler/rustc_public/src/mir/body.rs
index 3320b98cd61..3d595286041 100644
--- a/compiler/rustc_public/src/mir/body.rs
+++ b/compiler/rustc_public/src/mir/body.rs
@@ -654,9 +654,7 @@ impl Rvalue {
                 )),
                 AggregateKind::Adt(def, _, ref args, _, _) => Ok(def.ty_with_args(args)),
                 AggregateKind::Closure(def, ref args) => Ok(Ty::new_closure(def, args.clone())),
-                AggregateKind::Coroutine(def, ref args, mov) => {
-                    Ok(Ty::new_coroutine(def, args.clone(), mov))
-                }
+                AggregateKind::Coroutine(def, ref args) => Ok(Ty::new_coroutine(def, args.clone())),
                 AggregateKind::CoroutineClosure(def, ref args) => {
                     Ok(Ty::new_coroutine_closure(def, args.clone()))
                 }
@@ -674,8 +672,7 @@ pub enum AggregateKind {
     Tuple,
     Adt(AdtDef, VariantIdx, GenericArgs, Option<UserTypeAnnotationIndex>, Option<FieldIdx>),
     Closure(ClosureDef, GenericArgs),
-    // FIXME(rustc_public): Movability here is redundant
-    Coroutine(CoroutineDef, GenericArgs, Movability),
+    Coroutine(CoroutineDef, GenericArgs),
     CoroutineClosure(CoroutineClosureDef, GenericArgs),
     RawPtr(Ty, Mutability),
 }
diff --git a/compiler/rustc_public/src/mir/pretty.rs b/compiler/rustc_public/src/mir/pretty.rs
index a433df2dba1..3183c020772 100644
--- a/compiler/rustc_public/src/mir/pretty.rs
+++ b/compiler/rustc_public/src/mir/pretty.rs
@@ -428,7 +428,7 @@ fn pretty_aggregate<W: Write>(
             write!(writer, "{{closure@{}}}(", def.span().diagnostic())?;
             ")"
         }
-        AggregateKind::Coroutine(def, _, _) => {
+        AggregateKind::Coroutine(def, _) => {
             write!(writer, "{{coroutine@{}}}(", def.span().diagnostic())?;
             ")"
         }
diff --git a/compiler/rustc_public/src/ty.rs b/compiler/rustc_public/src/ty.rs
index bc67a2f987d..de4b21b1764 100644
--- a/compiler/rustc_public/src/ty.rs
+++ b/compiler/rustc_public/src/ty.rs
@@ -60,8 +60,8 @@ impl Ty {
     }
 
     /// Create a new coroutine type.
-    pub fn new_coroutine(def: CoroutineDef, args: GenericArgs, mov: Movability) -> Ty {
-        Ty::from_rigid_kind(RigidTy::Coroutine(def, args, mov))
+    pub fn new_coroutine(def: CoroutineDef, args: GenericArgs) -> Ty {
+        Ty::from_rigid_kind(RigidTy::Coroutine(def, args))
     }
 
     /// Create a new closure type.
@@ -560,8 +560,7 @@ pub enum RigidTy {
     FnDef(FnDef, GenericArgs),
     FnPtr(PolyFnSig),
     Closure(ClosureDef, GenericArgs),
-    // FIXME(rustc_public): Movability here is redundant
-    Coroutine(CoroutineDef, GenericArgs, Movability),
+    Coroutine(CoroutineDef, GenericArgs),
     CoroutineClosure(CoroutineClosureDef, GenericArgs),
     Dynamic(Vec<Binder<ExistentialPredicate>>, Region, DynKind),
     Never,
diff --git a/compiler/rustc_public/src/unstable/convert/internal.rs b/compiler/rustc_public/src/unstable/convert/internal.rs
index b2d38e497bc..66f767a98f5 100644
--- a/compiler/rustc_public/src/unstable/convert/internal.rs
+++ b/compiler/rustc_public/src/unstable/convert/internal.rs
@@ -177,7 +177,7 @@ impl RustcInternal for RigidTy {
             RigidTy::Closure(def, args) => {
                 rustc_ty::TyKind::Closure(def.0.internal(tables, tcx), args.internal(tables, tcx))
             }
-            RigidTy::Coroutine(def, args, _mov) => {
+            RigidTy::Coroutine(def, args) => {
                 rustc_ty::TyKind::Coroutine(def.0.internal(tables, tcx), args.internal(tables, tcx))
             }
             RigidTy::CoroutineClosure(def, args) => rustc_ty::TyKind::CoroutineClosure(
diff --git a/compiler/rustc_public/src/unstable/convert/stable/mir.rs b/compiler/rustc_public/src/unstable/convert/stable/mir.rs
index 8dee579e598..be8ee80bed3 100644
--- a/compiler/rustc_public/src/unstable/convert/stable/mir.rs
+++ b/compiler/rustc_public/src/unstable/convert/stable/mir.rs
@@ -653,7 +653,6 @@ impl<'tcx> Stable<'tcx> for mir::AggregateKind<'tcx> {
                 crate::mir::AggregateKind::Coroutine(
                     tables.coroutine_def(*def_id),
                     generic_arg.stable(tables, cx),
-                    cx.coroutine_movability(*def_id).stable(tables, cx),
                 )
             }
             mir::AggregateKind::CoroutineClosure(def_id, generic_args) => {
diff --git a/compiler/rustc_public/src/unstable/convert/stable/ty.rs b/compiler/rustc_public/src/unstable/convert/stable/ty.rs
index d679615b3bd..5a661072bc7 100644
--- a/compiler/rustc_public/src/unstable/convert/stable/ty.rs
+++ b/compiler/rustc_public/src/unstable/convert/stable/ty.rs
@@ -457,7 +457,6 @@ impl<'tcx> Stable<'tcx> for ty::TyKind<'tcx> {
             ty::Coroutine(def_id, generic_args) => TyKind::RigidTy(RigidTy::Coroutine(
                 tables.coroutine_def(*def_id),
                 generic_args.stable(tables, cx),
-                cx.coroutine_movability(*def_id).stable(tables, cx),
             )),
             ty::Never => TyKind::RigidTy(RigidTy::Never),
             ty::Tuple(fields) => TyKind::RigidTy(RigidTy::Tuple(
diff --git a/compiler/rustc_public/src/visitor.rs b/compiler/rustc_public/src/visitor.rs
index 45e2a815470..87f1cc6ae69 100644
--- a/compiler/rustc_public/src/visitor.rs
+++ b/compiler/rustc_public/src/visitor.rs
@@ -166,7 +166,7 @@ impl Visitable for RigidTy {
             }
             RigidTy::Adt(_, args)
             | RigidTy::Closure(_, args)
-            | RigidTy::Coroutine(_, args, _)
+            | RigidTy::Coroutine(_, args)
             | RigidTy::CoroutineWitness(_, args)
             | RigidTy::CoroutineClosure(_, args)
             | RigidTy::FnDef(_, args) => args.visit(visitor),
diff --git a/compiler/rustc_public_bridge/src/alloc.rs b/compiler/rustc_public_bridge/src/alloc.rs
index ecf9004562c..7e6af342546 100644
--- a/compiler/rustc_public_bridge/src/alloc.rs
+++ b/compiler/rustc_public_bridge/src/alloc.rs
@@ -38,11 +38,10 @@ pub fn try_new_scalar<'tcx, B: Bridge>(
 
 pub fn try_new_slice<'tcx, B: Bridge>(
     layout: TyAndLayout<'tcx, Ty<'tcx>>,
-    data: ConstAllocation<'tcx>,
+    alloc_id: AllocId,
     meta: u64,
     cx: &CompilerCtxt<'tcx, B>,
 ) -> Result<Allocation, B::Error> {
-    let alloc_id = cx.tcx.reserve_and_set_memory_alloc(data);
     let ptr = Pointer::new(alloc_id.into(), Size::ZERO);
     let scalar_ptr = Scalar::from_pointer(ptr, &cx.tcx);
     let scalar_meta: Scalar = Scalar::from_target_usize(meta, &cx.tcx);
diff --git a/compiler/rustc_public_bridge/src/context/impls.rs b/compiler/rustc_public_bridge/src/context/impls.rs
index 612e44b56b1..9b3948d232d 100644
--- a/compiler/rustc_public_bridge/src/context/impls.rs
+++ b/compiler/rustc_public_bridge/src/context/impls.rs
@@ -63,7 +63,7 @@ impl<'tcx, B: Bridge> CompilerCtxt<'tcx, B> {
         self.tcx.coroutine_movability(def_id)
     }
 
-    pub fn valtree_to_const_val(&self, key: ty::Value<'tcx>) -> ConstValue<'tcx> {
+    pub fn valtree_to_const_val(&self, key: ty::Value<'tcx>) -> ConstValue {
         self.tcx.valtree_to_const_val(key)
     }
 
@@ -675,10 +675,7 @@ impl<'tcx, B: Bridge> CompilerCtxt<'tcx, B> {
     }
 
     /// Try to evaluate an instance into a constant.
-    pub fn eval_instance(
-        &self,
-        instance: ty::Instance<'tcx>,
-    ) -> Result<ConstValue<'tcx>, ErrorHandled> {
+    pub fn eval_instance(&self, instance: ty::Instance<'tcx>) -> Result<ConstValue, ErrorHandled> {
         self.tcx.const_eval_instance(
             self.fully_monomorphized(),
             instance,
diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs
index 06e59eb4ccc..e74de5edc42 100644
--- a/compiler/rustc_query_system/src/query/plumbing.rs
+++ b/compiler/rustc_query_system/src/query/plumbing.rs
@@ -720,7 +720,7 @@ fn incremental_verify_ich_failed<Tcx>(
         static INSIDE_VERIFY_PANIC: Cell<bool> = const { Cell::new(false) };
     };
 
-    let old_in_panic = INSIDE_VERIFY_PANIC.with(|in_panic| in_panic.replace(true));
+    let old_in_panic = INSIDE_VERIFY_PANIC.replace(true);
 
     if old_in_panic {
         tcx.sess().dcx().emit_err(crate::error::Reentrant);
@@ -739,7 +739,7 @@ fn incremental_verify_ich_failed<Tcx>(
         panic!("Found unstable fingerprints for {dep_node:?}: {}", result());
     }
 
-    INSIDE_VERIFY_PANIC.with(|in_panic| in_panic.set(old_in_panic));
+    INSIDE_VERIFY_PANIC.set(old_in_panic);
 }
 
 /// Ensure that either this query has all green inputs or been executed.
diff --git a/compiler/rustc_resolve/src/build_reduced_graph.rs b/compiler/rustc_resolve/src/build_reduced_graph.rs
index fe7fa71a7fc..b15b478a6be 100644
--- a/compiler/rustc_resolve/src/build_reduced_graph.rs
+++ b/compiler/rustc_resolve/src/build_reduced_graph.rs
@@ -23,9 +23,9 @@ use rustc_hir::def::{self, *};
 use rustc_hir::def_id::{CRATE_DEF_ID, DefId, LocalDefId};
 use rustc_index::bit_set::DenseBitSet;
 use rustc_metadata::creader::LoadedMacro;
-use rustc_middle::bug;
 use rustc_middle::metadata::ModChild;
 use rustc_middle::ty::{Feed, Visibility};
+use rustc_middle::{bug, span_bug};
 use rustc_span::hygiene::{ExpnId, LocalExpnId, MacroKind};
 use rustc_span::{Ident, Span, Symbol, kw, sym};
 use thin_vec::ThinVec;
@@ -46,31 +46,59 @@ type Res = def::Res<NodeId>;
 impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
     /// Defines `name` in namespace `ns` of module `parent` to be `def` if it is not yet defined;
     /// otherwise, reports an error.
-    pub(crate) fn define_binding(
+    pub(crate) fn define_binding_local(
         &mut self,
         parent: Module<'ra>,
         ident: Ident,
         ns: Namespace,
         binding: NameBinding<'ra>,
     ) {
-        let key = self.new_disambiguated_key(ident, ns);
-        if let Err(old_binding) = self.try_define(parent, key, binding, false) {
+        if let Err(old_binding) = self.try_define_local(parent, ident, ns, binding, false) {
             self.report_conflict(parent, ident, ns, old_binding, binding);
         }
     }
 
-    fn define(
+    fn define_local(
         &mut self,
         parent: Module<'ra>,
         ident: Ident,
         ns: Namespace,
         res: Res,
-        vis: Visibility<impl Into<DefId>>,
+        vis: Visibility,
         span: Span,
         expn_id: LocalExpnId,
     ) {
         let binding = self.arenas.new_res_binding(res, vis.to_def_id(), span, expn_id);
-        self.define_binding(parent, ident, ns, binding)
+        self.define_binding_local(parent, ident, ns, binding);
+    }
+
+    fn define_extern(
+        &self,
+        parent: Module<'ra>,
+        ident: Ident,
+        ns: Namespace,
+        res: Res,
+        vis: Visibility<DefId>,
+        span: Span,
+        expn_id: LocalExpnId,
+    ) {
+        let binding = self.arenas.new_res_binding(res, vis, span, expn_id);
+        // Even if underscore names cannot be looked up, we still need to add them to modules,
+        // because they can be fetched by glob imports from those modules, and bring traits
+        // into scope both directly and through glob imports.
+        let key = BindingKey::new_disambiguated(ident, ns, || {
+            parent.underscore_disambiguator.update(|d| d + 1);
+            parent.underscore_disambiguator.get()
+        });
+        if self
+            .resolution_or_default(parent, key)
+            .borrow_mut()
+            .non_glob_binding
+            .replace(binding)
+            .is_some()
+        {
+            span_bug!(span, "an external binding was already defined");
+        }
     }
 
     /// Walks up the tree of definitions starting at `def_id`,
@@ -193,7 +221,7 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
         visitor.parent_scope.macro_rules
     }
 
-    pub(crate) fn build_reduced_graph_external(&mut self, module: Module<'ra>) {
+    pub(crate) fn build_reduced_graph_external(&self, module: Module<'ra>) {
         for child in self.tcx.module_children(module.def_id()) {
             let parent_scope = ParentScope::module(module, self);
             self.build_reduced_graph_for_external_crate_res(child, parent_scope)
@@ -202,7 +230,7 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
 
     /// Builds the reduced graph for a single item in an external crate.
     fn build_reduced_graph_for_external_crate_res(
-        &mut self,
+        &self,
         child: &ModChild,
         parent_scope: ParentScope<'ra>,
     ) {
@@ -233,7 +261,7 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
                 _,
             )
             | Res::PrimTy(..)
-            | Res::ToolMod => self.define(parent, ident, TypeNS, res, vis, span, expansion),
+            | Res::ToolMod => self.define_extern(parent, ident, TypeNS, res, vis, span, expansion),
             Res::Def(
                 DefKind::Fn
                 | DefKind::AssocFn
@@ -242,9 +270,9 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
                 | DefKind::AssocConst
                 | DefKind::Ctor(..),
                 _,
-            ) => self.define(parent, ident, ValueNS, res, vis, span, expansion),
+            ) => self.define_extern(parent, ident, ValueNS, res, vis, span, expansion),
             Res::Def(DefKind::Macro(..), _) | Res::NonMacroAttr(..) => {
-                self.define(parent, ident, MacroNS, res, vis, span, expansion)
+                self.define_extern(parent, ident, MacroNS, res, vis, span, expansion)
             }
             Res::Def(
                 DefKind::TyParam
@@ -446,16 +474,20 @@ impl<'a, 'ra, 'tcx> BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
 
         self.r.indeterminate_imports.push(import);
         match import.kind {
-            // Don't add unresolved underscore imports to modules
-            ImportKind::Single { target: Ident { name: kw::Underscore, .. }, .. } => {}
             ImportKind::Single { target, type_ns_only, .. } => {
-                self.r.per_ns(|this, ns| {
-                    if !type_ns_only || ns == TypeNS {
-                        let key = BindingKey::new(target, ns);
-                        let mut resolution = this.resolution(current_module, key).borrow_mut();
-                        resolution.single_imports.insert(import);
-                    }
-                });
+                // Don't add underscore imports to `single_imports`
+                // because they cannot define any usable names.
+                if target.name != kw::Underscore {
+                    self.r.per_ns(|this, ns| {
+                        if !type_ns_only || ns == TypeNS {
+                            let key = BindingKey::new(target, ns);
+                            this.resolution_or_default(current_module, key)
+                                .borrow_mut()
+                                .single_imports
+                                .insert(import);
+                        }
+                    });
+                }
             }
             // We don't add prelude imports to the globs since they only affect lexical scopes,
             // which are not relevant to import resolution.
@@ -708,7 +740,7 @@ impl<'a, 'ra, 'tcx> BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
         let expansion = parent_scope.expansion;
 
         // Define a name in the type namespace if it is not anonymous.
-        self.r.define(parent, ident, TypeNS, adt_res, adt_vis, adt_span, expansion);
+        self.r.define_local(parent, ident, TypeNS, adt_res, adt_vis, adt_span, expansion);
         self.r.feed_visibility(feed, adt_vis);
         let def_id = feed.key();
 
@@ -760,7 +792,7 @@ impl<'a, 'ra, 'tcx> BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
             }
 
             ItemKind::Mod(_, ident, ref mod_kind) => {
-                self.r.define(parent, ident, TypeNS, res, vis, sp, expansion);
+                self.r.define_local(parent, ident, TypeNS, res, vis, sp, expansion);
 
                 if let ast::ModKind::Loaded(_, _, _, Err(_)) = mod_kind {
                     self.r.mods_with_parse_errors.insert(def_id);
@@ -779,10 +811,10 @@ impl<'a, 'ra, 'tcx> BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
             ItemKind::Const(box ConstItem { ident, .. })
             | ItemKind::Delegation(box Delegation { ident, .. })
             | ItemKind::Static(box StaticItem { ident, .. }) => {
-                self.r.define(parent, ident, ValueNS, res, vis, sp, expansion);
+                self.r.define_local(parent, ident, ValueNS, res, vis, sp, expansion);
             }
             ItemKind::Fn(box Fn { ident, .. }) => {
-                self.r.define(parent, ident, ValueNS, res, vis, sp, expansion);
+                self.r.define_local(parent, ident, ValueNS, res, vis, sp, expansion);
 
                 // Functions introducing procedural macros reserve a slot
                 // in the macro namespace as well (see #52225).
@@ -791,11 +823,11 @@ impl<'a, 'ra, 'tcx> BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
 
             // These items live in the type namespace.
             ItemKind::TyAlias(box TyAlias { ident, .. }) | ItemKind::TraitAlias(ident, ..) => {
-                self.r.define(parent, ident, TypeNS, res, vis, sp, expansion);
+                self.r.define_local(parent, ident, TypeNS, res, vis, sp, expansion);
             }
 
             ItemKind::Enum(ident, _, _) | ItemKind::Trait(box ast::Trait { ident, .. }) => {
-                self.r.define(parent, ident, TypeNS, res, vis, sp, expansion);
+                self.r.define_local(parent, ident, TypeNS, res, vis, sp, expansion);
 
                 self.parent_scope.module = self.r.new_local_module(
                     Some(parent),
@@ -847,7 +879,7 @@ impl<'a, 'ra, 'tcx> BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
                     let feed = self.r.feed(ctor_node_id);
                     let ctor_def_id = feed.key();
                     let ctor_res = self.res(ctor_def_id);
-                    self.r.define(parent, ident, ValueNS, ctor_res, ctor_vis, sp, expansion);
+                    self.r.define_local(parent, ident, ValueNS, ctor_res, ctor_vis, sp, expansion);
                     self.r.feed_visibility(feed, ctor_vis);
                     // We need the field visibility spans also for the constructor for E0603.
                     self.insert_field_visibilities_local(ctor_def_id.to_def_id(), vdata.fields());
@@ -971,7 +1003,7 @@ impl<'a, 'ra, 'tcx> BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
                 );
             }
         }
-        self.r.define_binding(parent, ident, TypeNS, imported_binding);
+        self.r.define_binding_local(parent, ident, TypeNS, imported_binding);
     }
 
     /// Constructs the reduced graph for one foreign item.
@@ -988,7 +1020,7 @@ impl<'a, 'ra, 'tcx> BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
         let parent = self.parent_scope.module;
         let expansion = self.parent_scope.expansion;
         let vis = self.resolve_visibility(&item.vis);
-        self.r.define(parent, ident, ns, self.res(def_id), vis, item.span, expansion);
+        self.r.define_local(parent, ident, ns, self.res(def_id), vis, item.span, expansion);
         self.r.feed_visibility(feed, vis);
     }
 
@@ -1071,7 +1103,7 @@ impl<'a, 'ra, 'tcx> BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
         if let Some(span) = import_all {
             let import = macro_use_import(self, span, false);
             self.r.potentially_unused_imports.push(import);
-            module.for_each_child(self, |this, ident, ns, binding| {
+            module.for_each_child_mut(self, |this, ident, ns, binding| {
                 if ns == MacroNS {
                     let import = if this.r.is_accessible_from(binding.vis, this.parent_scope.module)
                     {
@@ -1236,7 +1268,7 @@ impl<'a, 'ra, 'tcx> BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
                 });
                 self.r.import_use_map.insert(import, Used::Other);
                 let import_binding = self.r.import(binding, import);
-                self.r.define_binding(self.r.graph_root, ident, MacroNS, import_binding);
+                self.r.define_binding_local(self.r.graph_root, ident, MacroNS, import_binding);
             } else {
                 self.r.check_reserved_macro_name(ident, res);
                 self.insert_unused_macro(ident, def_id, item.id);
@@ -1264,7 +1296,7 @@ impl<'a, 'ra, 'tcx> BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
             if !vis.is_public() {
                 self.insert_unused_macro(ident, def_id, item.id);
             }
-            self.r.define(module, ident, MacroNS, res, vis, span, expansion);
+            self.r.define_local(module, ident, MacroNS, res, vis, span, expansion);
             self.r.feed_visibility(feed, vis);
             self.parent_scope.macro_rules
         }
@@ -1400,10 +1432,13 @@ impl<'a, 'ra, 'tcx> Visitor<'a> for BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
         if ctxt == AssocCtxt::Trait {
             let parent = self.parent_scope.module;
             let expansion = self.parent_scope.expansion;
-            self.r.define(parent, ident, ns, self.res(def_id), vis, item.span, expansion);
-        } else if !matches!(&item.kind, AssocItemKind::Delegation(deleg) if deleg.from_glob) {
+            self.r.define_local(parent, ident, ns, self.res(def_id), vis, item.span, expansion);
+        } else if !matches!(&item.kind, AssocItemKind::Delegation(deleg) if deleg.from_glob)
+            && ident.name != kw::Underscore
+        {
+            // Don't add underscore names, they cannot be looked up anyway.
             let impl_def_id = self.r.tcx.local_parent(local_def_id);
-            let key = BindingKey::new(ident.normalize_to_macros_2_0(), ns);
+            let key = BindingKey::new(ident, ns);
             self.r.impl_binding_keys.entry(impl_def_id).or_default().insert(key);
         }
 
@@ -1485,7 +1520,7 @@ impl<'a, 'ra, 'tcx> Visitor<'a> for BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
         let feed = self.r.feed(variant.id);
         let def_id = feed.key();
         let vis = self.resolve_visibility(&variant.vis);
-        self.r.define(parent, ident, TypeNS, self.res(def_id), vis, variant.span, expn_id);
+        self.r.define_local(parent, ident, TypeNS, self.res(def_id), vis, variant.span, expn_id);
         self.r.feed_visibility(feed, vis);
 
         // If the variant is marked as non_exhaustive then lower the visibility to within the crate.
@@ -1501,7 +1536,7 @@ impl<'a, 'ra, 'tcx> Visitor<'a> for BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
             let feed = self.r.feed(ctor_node_id);
             let ctor_def_id = feed.key();
             let ctor_res = self.res(ctor_def_id);
-            self.r.define(parent, ident, ValueNS, ctor_res, ctor_vis, variant.span, expn_id);
+            self.r.define_local(parent, ident, ValueNS, ctor_res, ctor_vis, variant.span, expn_id);
             self.r.feed_visibility(feed, ctor_vis);
         }
 
diff --git a/compiler/rustc_resolve/src/check_unused.rs b/compiler/rustc_resolve/src/check_unused.rs
index 81ee02ac3c7..b85a814776a 100644
--- a/compiler/rustc_resolve/src/check_unused.rs
+++ b/compiler/rustc_resolve/src/check_unused.rs
@@ -509,9 +509,7 @@ impl Resolver<'_, '_> {
         let mut check_redundant_imports = FxIndexSet::default();
         for module in self.arenas.local_modules().iter() {
             for (_key, resolution) in self.resolutions(*module).borrow().iter() {
-                let resolution = resolution.borrow();
-
-                if let Some(binding) = resolution.best_binding()
+                if let Some(binding) = resolution.borrow().best_binding()
                     && let NameBindingKind::Import { import, .. } = binding.kind
                     && let ImportKind::Single { id, .. } = import.kind
                 {
diff --git a/compiler/rustc_resolve/src/diagnostics.rs b/compiler/rustc_resolve/src/diagnostics.rs
index d72fbc189e7..3af69b28780 100644
--- a/compiler/rustc_resolve/src/diagnostics.rs
+++ b/compiler/rustc_resolve/src/diagnostics.rs
@@ -523,7 +523,7 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
     }
 
     pub(crate) fn add_module_candidates(
-        &mut self,
+        &self,
         module: Module<'ra>,
         names: &mut Vec<TypoSuggestion>,
         filter_fn: &impl Fn(Res) -> bool,
@@ -1076,11 +1076,6 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
                         }
                     }
                 }
-                Scope::CrateRoot => {
-                    let root_ident = Ident::new(kw::PathRoot, ident.span);
-                    let root_module = this.resolve_crate_root(root_ident);
-                    this.add_module_candidates(root_module, &mut suggestions, filter_fn, None);
-                }
                 Scope::Module(module, _) => {
                     this.add_module_candidates(module, &mut suggestions, filter_fn, None);
                 }
@@ -1155,7 +1150,7 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
     }
 
     fn lookup_import_candidates_from_module<FilterFn>(
-        &mut self,
+        &self,
         lookup_ident: Ident,
         namespace: Namespace,
         parent_scope: &ParentScope<'ra>,
@@ -2664,10 +2659,8 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
             return None;
         }
 
-        let resolutions = self.resolutions(crate_module).borrow();
         let binding_key = BindingKey::new(ident, MacroNS);
-        let resolution = resolutions.get(&binding_key)?;
-        let binding = resolution.borrow().binding()?;
+        let binding = self.resolution(crate_module, binding_key)?.binding()?;
         let Res::Def(DefKind::Macro(MacroKind::Bang), _) = binding.res() else {
             return None;
         };
diff --git a/compiler/rustc_resolve/src/effective_visibilities.rs b/compiler/rustc_resolve/src/effective_visibilities.rs
index 34d1e9552fd..fe6e5b8e6eb 100644
--- a/compiler/rustc_resolve/src/effective_visibilities.rs
+++ b/compiler/rustc_resolve/src/effective_visibilities.rs
@@ -114,9 +114,7 @@ impl<'a, 'ra, 'tcx> EffectiveVisibilitiesVisitor<'a, 'ra, 'tcx> {
     /// including their whole reexport chains.
     fn set_bindings_effective_visibilities(&mut self, module_id: LocalDefId) {
         let module = self.r.expect_module(module_id.to_def_id());
-        let resolutions = self.r.resolutions(module);
-
-        for (_, name_resolution) in resolutions.borrow().iter() {
+        for (_, name_resolution) in self.r.resolutions(module).borrow().iter() {
             let Some(mut binding) = name_resolution.borrow().binding() else {
                 continue;
             };
diff --git a/compiler/rustc_resolve/src/ident.rs b/compiler/rustc_resolve/src/ident.rs
index 34941398a2b..f5bc46bf053 100644
--- a/compiler/rustc_resolve/src/ident.rs
+++ b/compiler/rustc_resolve/src/ident.rs
@@ -93,20 +93,21 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
         // 6. Language prelude: builtin attributes (closed, controlled).
 
         let rust_2015 = ctxt.edition().is_rust_2015();
-        let (ns, macro_kind, is_absolute_path) = match scope_set {
-            ScopeSet::All(ns) => (ns, None, false),
-            ScopeSet::AbsolutePath(ns) => (ns, None, true),
-            ScopeSet::Macro(macro_kind) => (MacroNS, Some(macro_kind), false),
-            ScopeSet::Late(ns, ..) => (ns, None, false),
+        let (ns, macro_kind) = match scope_set {
+            ScopeSet::All(ns)
+            | ScopeSet::ModuleAndExternPrelude(ns, _)
+            | ScopeSet::Late(ns, ..) => (ns, None),
+            ScopeSet::Macro(macro_kind) => (MacroNS, Some(macro_kind)),
         };
         let module = match scope_set {
             // Start with the specified module.
-            ScopeSet::Late(_, module, _) => module,
+            ScopeSet::Late(_, module, _) | ScopeSet::ModuleAndExternPrelude(_, module) => module,
             // Jump out of trait or enum modules, they do not act as scopes.
             _ => parent_scope.module.nearest_item_scope(),
         };
+        let module_and_extern_prelude = matches!(scope_set, ScopeSet::ModuleAndExternPrelude(..));
         let mut scope = match ns {
-            _ if is_absolute_path => Scope::CrateRoot,
+            _ if module_and_extern_prelude => Scope::Module(module, None),
             TypeNS | ValueNS => Scope::Module(module, None),
             MacroNS => Scope::DeriveHelpers(parent_scope.expansion),
         };
@@ -134,11 +135,10 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
                     }
                     true
                 }
-                Scope::CrateRoot => true,
                 Scope::Module(..) => true,
                 Scope::MacroUsePrelude => use_prelude || rust_2015,
                 Scope::BuiltinAttrs => true,
-                Scope::ExternPrelude => use_prelude || is_absolute_path,
+                Scope::ExternPrelude => use_prelude || module_and_extern_prelude,
                 Scope::ToolPrelude => use_prelude,
                 Scope::StdLibPrelude => use_prelude || ns == MacroNS,
                 Scope::BuiltinTypes => true,
@@ -174,7 +174,7 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
                     }
                     MacroRulesScope::Empty => Scope::Module(module, None),
                 },
-                Scope::CrateRoot => match ns {
+                Scope::Module(..) if module_and_extern_prelude => match ns {
                     TypeNS => {
                         ctxt.adjust(ExpnId::root());
                         Scope::ExternPrelude
@@ -203,7 +203,7 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
                 }
                 Scope::MacroUsePrelude => Scope::StdLibPrelude,
                 Scope::BuiltinAttrs => break, // nowhere else to search
-                Scope::ExternPrelude if is_absolute_path => break,
+                Scope::ExternPrelude if module_and_extern_prelude => break,
                 Scope::ExternPrelude => Scope::ToolPrelude,
                 Scope::ToolPrelude => Scope::StdLibPrelude,
                 Scope::StdLibPrelude => match ns {
@@ -404,10 +404,10 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
         }
 
         let (ns, macro_kind) = match scope_set {
-            ScopeSet::All(ns) => (ns, None),
-            ScopeSet::AbsolutePath(ns) => (ns, None),
+            ScopeSet::All(ns)
+            | ScopeSet::ModuleAndExternPrelude(ns, _)
+            | ScopeSet::Late(ns, ..) => (ns, None),
             ScopeSet::Macro(macro_kind) => (MacroNS, Some(macro_kind)),
-            ScopeSet::Late(ns, ..) => (ns, None),
         };
 
         // This is *the* result, resolution from the scope closest to the resolved identifier.
@@ -487,31 +487,16 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
                         MacroRulesScope::Invocation(_) => Err(Determinacy::Undetermined),
                         _ => Err(Determinacy::Determined),
                     },
-                    Scope::CrateRoot => {
-                        let root_ident = Ident::new(kw::PathRoot, ident.span);
-                        let root_module = this.resolve_crate_root(root_ident);
-                        let binding = this.resolve_ident_in_module(
-                            ModuleOrUniformRoot::Module(root_module),
-                            ident,
-                            ns,
-                            parent_scope,
-                            finalize,
-                            ignore_binding,
-                            ignore_import,
-                        );
-                        match binding {
-                            Ok(binding) => Ok((binding, Flags::MODULE | Flags::MISC_SUGGEST_CRATE)),
-                            Err((Determinacy::Undetermined, Weak::No)) => {
-                                return Some(Err(Determinacy::determined(force)));
-                            }
-                            Err((Determinacy::Undetermined, Weak::Yes)) => {
-                                Err(Determinacy::Undetermined)
-                            }
-                            Err((Determinacy::Determined, _)) => Err(Determinacy::Determined),
-                        }
-                    }
                     Scope::Module(module, derive_fallback_lint_id) => {
-                        let adjusted_parent_scope = &ParentScope { module, ..*parent_scope };
+                        let (adjusted_parent_scope, finalize) =
+                            if matches!(scope_set, ScopeSet::ModuleAndExternPrelude(..)) {
+                                (parent_scope, finalize)
+                            } else {
+                                (
+                                    &ParentScope { module, ..*parent_scope },
+                                    finalize.map(|f| Finalize { used: Used::Scope, ..f }),
+                                )
+                            };
                         let binding = this.resolve_ident_in_module_unadjusted(
                             ModuleOrUniformRoot::Module(module),
                             ident,
@@ -522,7 +507,7 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
                             } else {
                                 Shadowing::Restricted
                             },
-                            finalize.map(|finalize| Finalize { used: Used::Scope, ..finalize }),
+                            finalize,
                             ignore_binding,
                             ignore_import,
                         );
@@ -776,7 +761,7 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
             ModuleOrUniformRoot::ExternPrelude => {
                 ident.span.normalize_to_macros_2_0_and_adjust(ExpnId::root());
             }
-            ModuleOrUniformRoot::CrateRootAndExternPrelude | ModuleOrUniformRoot::CurrentScope => {
+            ModuleOrUniformRoot::ModuleAndExternPrelude(..) | ModuleOrUniformRoot::CurrentScope => {
                 // No adjustments
             }
         }
@@ -810,11 +795,11 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
     ) -> Result<NameBinding<'ra>, (Determinacy, Weak)> {
         let module = match module {
             ModuleOrUniformRoot::Module(module) => module,
-            ModuleOrUniformRoot::CrateRootAndExternPrelude => {
+            ModuleOrUniformRoot::ModuleAndExternPrelude(module) => {
                 assert_eq!(shadowing, Shadowing::Unrestricted);
                 let binding = self.early_resolve_ident_in_lexical_scope(
                     ident,
-                    ScopeSet::AbsolutePath(ns),
+                    ScopeSet::ModuleAndExternPrelude(ns, module),
                     parent_scope,
                     finalize,
                     finalize.is_some(),
@@ -863,8 +848,13 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
         };
 
         let key = BindingKey::new(ident, ns);
-        let resolution =
-            self.resolution(module, key).try_borrow_mut().map_err(|_| (Determined, Weak::No))?; // This happens when there is a cycle of imports.
+        // `try_borrow_mut` is required to ensure exclusive access, even if the resulting binding
+        // doesn't need to be mutable. It will fail when there is a cycle of imports, and without
+        // the exclusive access infinite recursion will crash the compiler with stack overflow.
+        let resolution = &*self
+            .resolution_or_default(module, key)
+            .try_borrow_mut()
+            .map_err(|_| (Determined, Weak::No))?;
 
         // If the primary binding is unusable, search further and return the shadowed glob
         // binding if it exists. What we really want here is having two separate scopes in
@@ -1531,7 +1521,8 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
                         && self.tcx.sess.at_least_rust_2018()
                     {
                         // `::a::b` from 2015 macro on 2018 global edition
-                        module = Some(ModuleOrUniformRoot::CrateRootAndExternPrelude);
+                        let crate_root = self.resolve_crate_root(ident);
+                        module = Some(ModuleOrUniformRoot::ModuleAndExternPrelude(crate_root));
                         continue;
                     }
                     if name == kw::PathRoot || name == kw::Crate || name == kw::DollarCrate {
diff --git a/compiler/rustc_resolve/src/imports.rs b/compiler/rustc_resolve/src/imports.rs
index 0a4c25b0eb0..156df45147f 100644
--- a/compiler/rustc_resolve/src/imports.rs
+++ b/compiler/rustc_resolve/src/imports.rs
@@ -25,7 +25,7 @@ use rustc_span::{Ident, Span, Symbol, kw, sym};
 use smallvec::SmallVec;
 use tracing::debug;
 
-use crate::Namespace::*;
+use crate::Namespace::{self, *};
 use crate::diagnostics::{DiagMode, Suggestion, import_candidates};
 use crate::errors::{
     CannotBeReexportedCratePublic, CannotBeReexportedCratePublicNS, CannotBeReexportedPrivate,
@@ -181,10 +181,10 @@ pub(crate) struct ImportData<'ra> {
     ///
     /// | `module_path` | `imported_module` | remark |
     /// |-|-|-|
-    /// |`use prefix::foo`| `ModuleOrUniformRoot::Module(prefix)`               | - |
-    /// |`use ::foo`      | `ModuleOrUniformRoot::ExternPrelude`                | 2018+ editions |
-    /// |`use ::foo`      | `ModuleOrUniformRoot::CrateRootAndExternPrelude`    | a special case in 2015 edition |
-    /// |`use foo`        | `ModuleOrUniformRoot::CurrentScope`                 | - |
+    /// |`use prefix::foo`| `ModuleOrUniformRoot::Module(prefix)`         | - |
+    /// |`use ::foo`      | `ModuleOrUniformRoot::ExternPrelude`          | 2018+ editions |
+    /// |`use ::foo`      | `ModuleOrUniformRoot::ModuleAndExternPrelude` | a special case in 2015 edition |
+    /// |`use foo`        | `ModuleOrUniformRoot::CurrentScope`           | - |
     pub imported_module: Cell<Option<ModuleOrUniformRoot<'ra>>>,
     pub vis: Visibility,
 }
@@ -334,18 +334,25 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
     }
 
     /// Define the name or return the existing binding if there is a collision.
-    /// `update` indicates if the definition is a redefinition of an existing binding.
-    pub(crate) fn try_define(
+    pub(crate) fn try_define_local(
         &mut self,
         module: Module<'ra>,
-        key: BindingKey,
+        ident: Ident,
+        ns: Namespace,
         binding: NameBinding<'ra>,
         warn_ambiguity: bool,
     ) -> Result<(), NameBinding<'ra>> {
         let res = binding.res();
-        self.check_reserved_macro_name(key.ident, res);
+        self.check_reserved_macro_name(ident, res);
         self.set_binding_parent_module(binding, module);
-        self.update_resolution(module, key, warn_ambiguity, |this, resolution| {
+        // Even if underscore names cannot be looked up, we still need to add them to modules,
+        // because they can be fetched by glob imports from those modules, and bring traits
+        // into scope both directly and through glob imports.
+        let key = BindingKey::new_disambiguated(ident, ns, || {
+            module.underscore_disambiguator.update(|d| d + 1);
+            module.underscore_disambiguator.get()
+        });
+        self.update_local_resolution(module, key, warn_ambiguity, |this, resolution| {
             if let Some(old_binding) = resolution.best_binding() {
                 if res == Res::Err && old_binding.res() != Res::Err {
                     // Do not override real bindings with `Res::Err`s from error recovery.
@@ -383,7 +390,7 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
                     (old_glob @ true, false) | (old_glob @ false, true) => {
                         let (glob_binding, non_glob_binding) =
                             if old_glob { (old_binding, binding) } else { (binding, old_binding) };
-                        if key.ns == MacroNS
+                        if ns == MacroNS
                             && non_glob_binding.expansion != LocalExpnId::ROOT
                             && glob_binding.res() != non_glob_binding.res()
                         {
@@ -448,7 +455,7 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
 
     // Use `f` to mutate the resolution of the name in the module.
     // If the resolution becomes a success, define it in the module's glob importers.
-    fn update_resolution<T, F>(
+    fn update_local_resolution<T, F>(
         &mut self,
         module: Module<'ra>,
         key: BindingKey,
@@ -461,7 +468,7 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
         // Ensure that `resolution` isn't borrowed when defining in the module's glob importers,
         // during which the resolution might end up getting re-defined via a glob cycle.
         let (binding, t, warn_ambiguity) = {
-            let resolution = &mut *self.resolution(module, key).borrow_mut();
+            let resolution = &mut *self.resolution_or_default(module, key).borrow_mut();
             let old_binding = resolution.binding();
 
             let t = f(self, resolution);
@@ -489,10 +496,10 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
             };
             if self.is_accessible_from(binding.vis, scope) {
                 let imported_binding = self.import(binding, *import);
-                let key = BindingKey { ident, ..key };
-                let _ = self.try_define(
+                let _ = self.try_define_local(
                     import.parent_scope.module,
-                    key,
+                    ident,
+                    key.ns,
                     imported_binding,
                     warn_ambiguity,
                 );
@@ -514,11 +521,15 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
             let dummy_binding = self.dummy_binding;
             let dummy_binding = self.import(dummy_binding, import);
             self.per_ns(|this, ns| {
-                let key = BindingKey::new(target, ns);
-                let _ = this.try_define(import.parent_scope.module, key, dummy_binding, false);
-                this.update_resolution(import.parent_scope.module, key, false, |_, resolution| {
-                    resolution.single_imports.swap_remove(&import);
-                })
+                let module = import.parent_scope.module;
+                let _ = this.try_define_local(module, target, ns, dummy_binding, false);
+                // Don't remove underscores from `single_imports`, they were never added.
+                if target.name != kw::Underscore {
+                    let key = BindingKey::new(target, ns);
+                    this.update_local_resolution(module, key, false, |_, resolution| {
+                        resolution.single_imports.swap_remove(&import);
+                    })
+                }
             });
             self.record_use(target, dummy_binding, Used::Other);
         } else if import.imported_module.get().is_none() {
@@ -639,7 +650,6 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
         for module in self.arenas.local_modules().iter() {
             for (key, resolution) in self.resolutions(*module).borrow().iter() {
                 let resolution = resolution.borrow();
-
                 let Some(binding) = resolution.best_binding() else { continue };
 
                 if let NameBindingKind::Import { import, .. } = binding.kind
@@ -891,14 +901,14 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
                         }
                         // We need the `target`, `source` can be extracted.
                         let imported_binding = this.import(binding, import);
-                        this.define_binding(parent, target, ns, imported_binding);
+                        this.define_binding_local(parent, target, ns, imported_binding);
                         PendingBinding::Ready(Some(imported_binding))
                     }
                     Err(Determinacy::Determined) => {
-                        // Don't update the resolution for underscores, because it was never added.
+                        // Don't remove underscores from `single_imports`, they were never added.
                         if target.name != kw::Underscore {
                             let key = BindingKey::new(target, ns);
-                            this.update_resolution(parent, key, false, |_, resolution| {
+                            this.update_local_resolution(parent, key, false, |_, resolution| {
                                 resolution.single_imports.swap_remove(&import);
                             });
                         }
@@ -1190,41 +1200,39 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
             });
 
             return if all_ns_failed {
-                let resolutions = match module {
-                    ModuleOrUniformRoot::Module(module) => Some(self.resolutions(module).borrow()),
-                    _ => None,
-                };
-                let resolutions = resolutions.as_ref().into_iter().flat_map(|r| r.iter());
-                let names = resolutions
-                    .filter_map(|(BindingKey { ident: i, .. }, resolution)| {
-                        if i.name == ident.name {
-                            return None;
-                        } // Never suggest the same name
-                        match *resolution.borrow() {
-                            ref resolution
-                                if let Some(name_binding) = resolution.best_binding() =>
-                            {
-                                match name_binding.kind {
-                                    NameBindingKind::Import { binding, .. } => {
-                                        match binding.kind {
-                                            // Never suggest the name that has binding error
-                                            // i.e., the name that cannot be previously resolved
-                                            NameBindingKind::Res(Res::Err) => None,
-                                            _ => Some(i.name),
+                let names = match module {
+                    ModuleOrUniformRoot::Module(module) => {
+                        self.resolutions(module)
+                            .borrow()
+                            .iter()
+                            .filter_map(|(BindingKey { ident: i, .. }, resolution)| {
+                                if i.name == ident.name {
+                                    return None;
+                                } // Never suggest the same name
+
+                                let resolution = resolution.borrow();
+                                if let Some(name_binding) = resolution.best_binding() {
+                                    match name_binding.kind {
+                                        NameBindingKind::Import { binding, .. } => {
+                                            match binding.kind {
+                                                // Never suggest the name that has binding error
+                                                // i.e., the name that cannot be previously resolved
+                                                NameBindingKind::Res(Res::Err) => None,
+                                                _ => Some(i.name),
+                                            }
                                         }
+                                        _ => Some(i.name),
                                     }
-                                    _ => Some(i.name),
+                                } else if resolution.single_imports.is_empty() {
+                                    None
+                                } else {
+                                    Some(i.name)
                                 }
-                            }
-                            NameResolution { ref single_imports, .. }
-                                if single_imports.is_empty() =>
-                            {
-                                None
-                            }
-                            _ => Some(i.name),
-                        }
-                    })
-                    .collect::<Vec<Symbol>>();
+                            })
+                            .collect()
+                    }
+                    _ => Vec::new(),
+                };
 
                 let lev_suggestion =
                     find_best_match_for_name(&names, ident.name, None).map(|suggestion| {
@@ -1505,12 +1513,12 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
                 let imported_binding = self.import(binding, import);
                 let warn_ambiguity = self
                     .resolution(import.parent_scope.module, key)
-                    .borrow()
-                    .binding()
+                    .and_then(|r| r.binding())
                     .is_some_and(|binding| binding.warn_ambiguity_recursive());
-                let _ = self.try_define(
+                let _ = self.try_define_local(
                     import.parent_scope.module,
-                    key,
+                    key.ident,
+                    key.ns,
                     imported_binding,
                     warn_ambiguity,
                 );
diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs
index a3a770502de..ed9622a0d81 100644
--- a/compiler/rustc_resolve/src/late.rs
+++ b/compiler/rustc_resolve/src/late.rs
@@ -3449,8 +3449,7 @@ impl<'a, 'ast, 'ra, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
         };
         ident.span.normalize_to_macros_2_0_and_adjust(module.expansion);
         let key = BindingKey::new(ident, ns);
-        let mut binding =
-            self.r.resolution(module, key).try_borrow().ok().and_then(|r| r.best_binding());
+        let mut binding = self.r.resolution(module, key).and_then(|r| r.best_binding());
         debug!(?binding);
         if binding.is_none() {
             // We could not find the trait item in the correct namespace.
@@ -3461,8 +3460,7 @@ impl<'a, 'ast, 'ra, 'tcx> LateResolutionVisitor<'a, 'ast, 'ra, 'tcx> {
                 _ => ns,
             };
             let key = BindingKey::new(ident, ns);
-            binding =
-                self.r.resolution(module, key).try_borrow().ok().and_then(|r| r.best_binding());
+            binding = self.r.resolution(module, key).and_then(|r| r.best_binding());
             debug!(?binding);
         }
 
diff --git a/compiler/rustc_resolve/src/late/diagnostics.rs b/compiler/rustc_resolve/src/late/diagnostics.rs
index 69095942f52..0f0babd4904 100644
--- a/compiler/rustc_resolve/src/late/diagnostics.rs
+++ b/compiler/rustc_resolve/src/late/diagnostics.rs
@@ -1461,15 +1461,17 @@ impl<'ast, 'ra, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
             if let PathResult::Module(ModuleOrUniformRoot::Module(module)) =
                 self.resolve_path(mod_path, None, None)
             {
-                let resolutions = self.r.resolutions(module).borrow();
-                let targets: Vec<_> = resolutions
+                let targets: Vec<_> = self
+                    .r
+                    .resolutions(module)
+                    .borrow()
                     .iter()
                     .filter_map(|(key, resolution)| {
                         resolution
                             .borrow()
                             .best_binding()
                             .map(|binding| binding.res())
-                            .and_then(|res| if filter_fn(res) { Some((key, res)) } else { None })
+                            .and_then(|res| if filter_fn(res) { Some((*key, res)) } else { None })
                     })
                     .collect();
                 if let [target] = targets.as_slice() {
@@ -2300,8 +2302,9 @@ impl<'ast, 'ra, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
             return None;
         }
 
-        let resolutions = self.r.resolutions(*module);
-        let targets = resolutions
+        let targets = self
+            .r
+            .resolutions(*module)
             .borrow()
             .iter()
             .filter_map(|(key, res)| {
@@ -2630,7 +2633,7 @@ impl<'ast, 'ra, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
         false
     }
 
-    fn find_module(&mut self, def_id: DefId) -> Option<(Module<'ra>, ImportSuggestion)> {
+    fn find_module(&self, def_id: DefId) -> Option<(Module<'ra>, ImportSuggestion)> {
         let mut result = None;
         let mut seen_modules = FxHashSet::default();
         let root_did = self.r.graph_root.def_id();
@@ -2687,7 +2690,7 @@ impl<'ast, 'ra, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
         result
     }
 
-    fn collect_enum_ctors(&mut self, def_id: DefId) -> Option<Vec<(Path, DefId, CtorKind)>> {
+    fn collect_enum_ctors(&self, def_id: DefId) -> Option<Vec<(Path, DefId, CtorKind)>> {
         self.find_module(def_id).map(|(enum_module, enum_import_suggestion)| {
             let mut variants = Vec::new();
             enum_module.for_each_child(self.r, |_, ident, _, name_binding| {
@@ -2704,7 +2707,7 @@ impl<'ast, 'ra, 'tcx> LateResolutionVisitor<'_, 'ast, 'ra, 'tcx> {
 
     /// Adds a suggestion for using an enum's variant when an enum is used instead.
     fn suggest_using_enum_variant(
-        &mut self,
+        &self,
         err: &mut Diag<'_>,
         source: PathSource<'_, '_, '_>,
         def_id: DefId,
diff --git a/compiler/rustc_resolve/src/lib.rs b/compiler/rustc_resolve/src/lib.rs
index 0d41a822e8a..8115b87dcae 100644
--- a/compiler/rustc_resolve/src/lib.rs
+++ b/compiler/rustc_resolve/src/lib.rs
@@ -21,7 +21,7 @@
 #![recursion_limit = "256"]
 // tidy-alphabetical-end
 
-use std::cell::{Cell, RefCell};
+use std::cell::{Cell, Ref, RefCell};
 use std::collections::BTreeSet;
 use std::fmt;
 use std::sync::Arc;
@@ -119,7 +119,6 @@ enum Scope<'ra> {
     DeriveHelpers(LocalExpnId),
     DeriveHelpersCompat,
     MacroRules(MacroRulesScopeRef<'ra>),
-    CrateRoot,
     // The node ID is for reporting the `PROC_MACRO_DERIVE_RESOLUTION_FALLBACK`
     // lint if it should be reported.
     Module(Module<'ra>, Option<NodeId>),
@@ -139,8 +138,8 @@ enum Scope<'ra> {
 enum ScopeSet<'ra> {
     /// All scopes with the given namespace.
     All(Namespace),
-    /// Crate root, then extern prelude (used for mixed 2015-2018 mode in macros).
-    AbsolutePath(Namespace),
+    /// A module, then extern prelude (used for mixed 2015-2018 mode in macros).
+    ModuleAndExternPrelude(Namespace, Module<'ra>),
     /// All scopes with macro namespace and the given macro kind restriction.
     Macro(MacroKind),
     /// All scopes with the given namespace, used for partially performing late resolution.
@@ -419,8 +418,10 @@ enum ModuleOrUniformRoot<'ra> {
     /// Regular module.
     Module(Module<'ra>),
 
-    /// Virtual module that denotes resolution in crate root with fallback to extern prelude.
-    CrateRootAndExternPrelude,
+    /// Virtual module that denotes resolution in a module with fallback to extern prelude.
+    /// Used for paths starting with `::` coming from 2015 edition macros
+    /// used in 2018+ edition crates.
+    ModuleAndExternPrelude(Module<'ra>),
 
     /// Virtual module that denotes resolution in extern prelude.
     /// Used for paths starting with `::` on 2018 edition.
@@ -532,15 +533,26 @@ struct BindingKey {
     /// identifier.
     ident: Ident,
     ns: Namespace,
-    /// 0 if ident is not `_`, otherwise a value that's unique to the specific
-    /// `_` in the expanded AST that introduced this binding.
+    /// When we add an underscore binding (with ident `_`) to some module, this field has
+    /// a non-zero value that uniquely identifies this binding in that module.
+    /// For non-underscore bindings this field is zero.
+    /// When a key is constructed for name lookup (as opposed to name definition), this field is
+    /// also zero, even for underscore names, so for underscores the lookup will never succeed.
     disambiguator: u32,
 }
 
 impl BindingKey {
     fn new(ident: Ident, ns: Namespace) -> Self {
-        let ident = ident.normalize_to_macros_2_0();
-        BindingKey { ident, ns, disambiguator: 0 }
+        BindingKey { ident: ident.normalize_to_macros_2_0(), ns, disambiguator: 0 }
+    }
+
+    fn new_disambiguated(
+        ident: Ident,
+        ns: Namespace,
+        disambiguator: impl FnOnce() -> u32,
+    ) -> BindingKey {
+        let disambiguator = if ident.name == kw::Underscore { disambiguator() } else { 0 };
+        BindingKey { ident: ident.normalize_to_macros_2_0(), ns, disambiguator }
     }
 }
 
@@ -568,6 +580,8 @@ struct ModuleData<'ra> {
     lazy_resolutions: Resolutions<'ra>,
     /// True if this is a module from other crate that needs to be populated on access.
     populate_on_access: Cell<bool>,
+    /// Used to disambiguate underscore items (`const _: T = ...`) in the module.
+    underscore_disambiguator: Cell<u32>,
 
     /// Macro invocations that can expand into items in this module.
     unexpanded_invocations: RefCell<FxHashSet<LocalExpnId>>,
@@ -628,6 +642,7 @@ impl<'ra> ModuleData<'ra> {
             kind,
             lazy_resolutions: Default::default(),
             populate_on_access: Cell::new(is_foreign),
+            underscore_disambiguator: Cell::new(0),
             unexpanded_invocations: Default::default(),
             no_implicit_prelude,
             glob_importers: RefCell::new(Vec::new()),
@@ -641,11 +656,23 @@ impl<'ra> ModuleData<'ra> {
 }
 
 impl<'ra> Module<'ra> {
-    fn for_each_child<'tcx, R, F>(self, resolver: &mut R, mut f: F)
-    where
-        R: AsMut<Resolver<'ra, 'tcx>>,
-        F: FnMut(&mut R, Ident, Namespace, NameBinding<'ra>),
-    {
+    fn for_each_child<'tcx, R: AsRef<Resolver<'ra, 'tcx>>>(
+        self,
+        resolver: &R,
+        mut f: impl FnMut(&R, Ident, Namespace, NameBinding<'ra>),
+    ) {
+        for (key, name_resolution) in resolver.as_ref().resolutions(self).borrow().iter() {
+            if let Some(binding) = name_resolution.borrow().best_binding() {
+                f(resolver, key.ident, key.ns, binding);
+            }
+        }
+    }
+
+    fn for_each_child_mut<'tcx, R: AsMut<Resolver<'ra, 'tcx>>>(
+        self,
+        resolver: &mut R,
+        mut f: impl FnMut(&mut R, Ident, Namespace, NameBinding<'ra>),
+    ) {
         for (key, name_resolution) in resolver.as_mut().resolutions(self).borrow().iter() {
             if let Some(binding) = name_resolution.borrow().best_binding() {
                 f(resolver, key.ident, key.ns, binding);
@@ -654,10 +681,7 @@ impl<'ra> Module<'ra> {
     }
 
     /// This modifies `self` in place. The traits will be stored in `self.traits`.
-    fn ensure_traits<'tcx, R>(self, resolver: &mut R)
-    where
-        R: AsMut<Resolver<'ra, 'tcx>>,
-    {
+    fn ensure_traits<'tcx>(self, resolver: &impl AsRef<Resolver<'ra, 'tcx>>) {
         let mut traits = self.traits.borrow_mut();
         if traits.is_none() {
             let mut collected_traits = Vec::new();
@@ -666,7 +690,7 @@ impl<'ra> Module<'ra> {
                     return;
                 }
                 if let Res::Def(DefKind::Trait | DefKind::TraitAlias, def_id) = binding.res() {
-                    collected_traits.push((name, binding, r.as_mut().get_module(def_id)))
+                    collected_traits.push((name, binding, r.as_ref().get_module(def_id)))
                 }
             });
             *traits = Some(collected_traits.into_boxed_slice());
@@ -1087,8 +1111,6 @@ pub struct Resolver<'ra, 'tcx> {
     extern_module_map: RefCell<FxIndexMap<DefId, Module<'ra>>>,
     binding_parent_modules: FxHashMap<NameBinding<'ra>, Module<'ra>>,
 
-    underscore_disambiguator: u32,
-
     /// Maps glob imports to the names of items actually imported.
     glob_map: FxIndexMap<LocalDefId, FxIndexSet<Symbol>>,
     glob_error: Option<ErrorGuaranteed>,
@@ -1327,6 +1349,12 @@ impl<'ra, 'tcx> AsMut<Resolver<'ra, 'tcx>> for Resolver<'ra, 'tcx> {
     }
 }
 
+impl<'ra, 'tcx> AsRef<Resolver<'ra, 'tcx>> for Resolver<'ra, 'tcx> {
+    fn as_ref(&self) -> &Resolver<'ra, 'tcx> {
+        self
+    }
+}
+
 impl<'tcx> Resolver<'_, 'tcx> {
     fn opt_local_def_id(&self, node: NodeId) -> Option<LocalDefId> {
         self.opt_feed(node).map(|f| f.key())
@@ -1500,7 +1528,6 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
             extern_crate_map: Default::default(),
             module_children: Default::default(),
             trait_map: NodeMap::default(),
-            underscore_disambiguator: 0,
             empty_module,
             local_module_map,
             extern_module_map: Default::default(),
@@ -1849,7 +1876,7 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
     // We don't reject trait aliases (`trait_module == None`) because we don't have access to their
     // associated items.
     fn trait_may_have_item(
-        &mut self,
+        &self,
         trait_module: Option<Module<'ra>>,
         assoc_item: Option<(Symbol, Namespace)>,
     ) -> bool {
@@ -1881,18 +1908,7 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
         import_ids
     }
 
-    fn new_disambiguated_key(&mut self, ident: Ident, ns: Namespace) -> BindingKey {
-        let ident = ident.normalize_to_macros_2_0();
-        let disambiguator = if ident.name == kw::Underscore {
-            self.underscore_disambiguator += 1;
-            self.underscore_disambiguator
-        } else {
-            0
-        };
-        BindingKey { ident, ns, disambiguator }
-    }
-
-    fn resolutions(&mut self, module: Module<'ra>) -> &'ra Resolutions<'ra> {
+    fn resolutions(&self, module: Module<'ra>) -> &'ra Resolutions<'ra> {
         if module.populate_on_access.get() {
             module.populate_on_access.set(false);
             self.build_reduced_graph_external(module);
@@ -1901,12 +1917,19 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
     }
 
     fn resolution(
-        &mut self,
+        &self,
+        module: Module<'ra>,
+        key: BindingKey,
+    ) -> Option<Ref<'ra, NameResolution<'ra>>> {
+        self.resolutions(module).borrow().get(&key).map(|resolution| resolution.borrow())
+    }
+
+    fn resolution_or_default(
+        &self,
         module: Module<'ra>,
         key: BindingKey,
     ) -> &'ra RefCell<NameResolution<'ra>> {
-        *self
-            .resolutions(module)
+        self.resolutions(module)
             .borrow_mut()
             .entry(key)
             .or_insert_with(|| self.arenas.alloc_name_resolution())
diff --git a/compiler/rustc_resolve/src/macros.rs b/compiler/rustc_resolve/src/macros.rs
index 333fc808d1b..b96d9e7dc8e 100644
--- a/compiler/rustc_resolve/src/macros.rs
+++ b/compiler/rustc_resolve/src/macros.rs
@@ -511,7 +511,7 @@ impl<'ra, 'tcx> ResolverExpand for Resolver<'ra, 'tcx> {
     }
 
     fn glob_delegation_suffixes(
-        &mut self,
+        &self,
         trait_def_id: DefId,
         impl_def_id: LocalDefId,
     ) -> Result<Vec<(Ident, Option<Ident>)>, Indeterminate> {
@@ -535,7 +535,7 @@ impl<'ra, 'tcx> ResolverExpand for Resolver<'ra, 'tcx> {
         target_trait.for_each_child(self, |this, ident, ns, _binding| {
             // FIXME: Adjust hygiene for idents from globs, like for glob imports.
             if let Some(overriding_keys) = this.impl_binding_keys.get(&impl_def_id)
-                && overriding_keys.contains(&BindingKey::new(ident.normalize_to_macros_2_0(), ns))
+                && overriding_keys.contains(&BindingKey::new(ident, ns))
             {
                 // The name is overridden, do not produce it from the glob delegation.
             } else {
diff --git a/compiler/rustc_resolve/src/rustdoc.rs b/compiler/rustc_resolve/src/rustdoc.rs
index 24e15ded94f..6450f63472c 100644
--- a/compiler/rustc_resolve/src/rustdoc.rs
+++ b/compiler/rustc_resolve/src/rustdoc.rs
@@ -509,9 +509,8 @@ fn collect_link_data<'input, F: BrokenLinkCallback<'input>>(
     display_text.map(String::into_boxed_str)
 }
 
-/// Returns a tuple containing a span encompassing all the document fragments and a boolean that is
-/// `true` if any of the fragments are from a macro expansion.
-pub fn span_of_fragments_with_expansion(fragments: &[DocFragment]) -> Option<(Span, bool)> {
+/// Returns a span encompassing all the document fragments.
+pub fn span_of_fragments(fragments: &[DocFragment]) -> Option<Span> {
     let (first_fragment, last_fragment) = match fragments {
         [] => return None,
         [first, .., last] => (first, last),
@@ -520,15 +519,7 @@ pub fn span_of_fragments_with_expansion(fragments: &[DocFragment]) -> Option<(Sp
     if first_fragment.span == DUMMY_SP {
         return None;
     }
-    Some((
-        first_fragment.span.to(last_fragment.span),
-        fragments.iter().any(|frag| frag.from_expansion),
-    ))
-}
-
-/// Returns a span encompassing all the document fragments.
-pub fn span_of_fragments(fragments: &[DocFragment]) -> Option<Span> {
-    span_of_fragments_with_expansion(fragments).map(|(sp, _)| sp)
+    Some(first_fragment.span.to(last_fragment.span))
 }
 
 /// Attempts to match a range of bytes from parsed markdown to a `Span` in the source code.
@@ -686,7 +677,7 @@ pub fn source_span_for_markdown_range_inner(
         }
     }
 
-    let (span, _) = span_of_fragments_with_expansion(fragments)?;
+    let span = span_of_fragments(fragments)?;
     let src_span = span.from_inner(InnerSpan::new(
         md_range.start + start_bytes,
         md_range.end + start_bytes + end_bytes,
diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs
index 7bea8685724..8f624e0fb2f 100644
--- a/compiler/rustc_session/src/config.rs
+++ b/compiler/rustc_session/src/config.rs
@@ -343,12 +343,12 @@ impl LinkSelfContained {
         if let Some(component_to_enable) = component.strip_prefix('+') {
             self.explicitly_set = None;
             self.enabled_components
-                .insert(LinkSelfContainedComponents::from_str(component_to_enable)?);
+                .insert(LinkSelfContainedComponents::from_str(component_to_enable).ok()?);
             Some(())
         } else if let Some(component_to_disable) = component.strip_prefix('-') {
             self.explicitly_set = None;
             self.disabled_components
-                .insert(LinkSelfContainedComponents::from_str(component_to_disable)?);
+                .insert(LinkSelfContainedComponents::from_str(component_to_disable).ok()?);
             Some(())
         } else {
             None
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index b33e3815ea4..5f1973b31a1 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -1296,7 +1296,7 @@ pub mod parse {
     }
 
     pub(crate) fn parse_linker_flavor(slot: &mut Option<LinkerFlavorCli>, v: Option<&str>) -> bool {
-        match v.and_then(LinkerFlavorCli::from_str) {
+        match v.and_then(|v| LinkerFlavorCli::from_str(v).ok()) {
             Some(lf) => *slot = Some(lf),
             _ => return false,
         }
diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs
index 85bd8340c3c..ddcdb7bbc18 100644
--- a/compiler/rustc_session/src/session.rs
+++ b/compiler/rustc_session/src/session.rs
@@ -313,6 +313,7 @@ impl Session {
             || self.opts.unstable_opts.query_dep_graph
             || self.opts.unstable_opts.dump_mir.is_some()
             || self.opts.unstable_opts.unpretty.is_some()
+            || self.prof.is_args_recording_enabled()
             || self.opts.output_types.contains_key(&OutputType::Mir)
             || std::env::var_os("RUSTC_LOG").is_some()
         {
diff --git a/compiler/rustc_span/src/hygiene.rs b/compiler/rustc_span/src/hygiene.rs
index c3080875da8..97d1d9c2d2a 100644
--- a/compiler/rustc_span/src/hygiene.rs
+++ b/compiler/rustc_span/src/hygiene.rs
@@ -322,6 +322,7 @@ impl ExpnId {
 
     /// `expn_id.outer_expn_is_descendant_of(ctxt)` is equivalent to but faster than
     /// `expn_id.is_descendant_of(ctxt.outer_expn())`.
+    #[inline]
     pub fn outer_expn_is_descendant_of(self, ctxt: SyntaxContext) -> bool {
         HygieneData::with(|data| data.is_descendant_of(self, data.outer_expn(ctxt)))
     }
@@ -394,6 +395,7 @@ impl HygieneData {
         }
     }
 
+    #[inline]
     fn with<R>(f: impl FnOnce(&mut HygieneData) -> R) -> R {
         with_session_globals(|session_globals| f(&mut session_globals.hygiene_data.borrow_mut()))
     }
@@ -406,6 +408,7 @@ impl HygieneData {
         }
     }
 
+    #[inline]
     fn local_expn_data(&self, expn_id: LocalExpnId) -> &ExpnData {
         self.local_expn_data[expn_id].as_ref().expect("no expansion data for an expansion ID")
     }
@@ -437,23 +440,28 @@ impl HygieneData {
         }
     }
 
+    #[inline]
     fn normalize_to_macros_2_0(&self, ctxt: SyntaxContext) -> SyntaxContext {
         self.syntax_context_data[ctxt.0 as usize].opaque
     }
 
+    #[inline]
     fn normalize_to_macro_rules(&self, ctxt: SyntaxContext) -> SyntaxContext {
         self.syntax_context_data[ctxt.0 as usize].opaque_and_semiopaque
     }
 
+    #[inline]
     fn outer_expn(&self, ctxt: SyntaxContext) -> ExpnId {
         self.syntax_context_data[ctxt.0 as usize].outer_expn
     }
 
+    #[inline]
     fn outer_mark(&self, ctxt: SyntaxContext) -> (ExpnId, Transparency) {
         let data = &self.syntax_context_data[ctxt.0 as usize];
         (data.outer_expn, data.outer_transparency)
     }
 
+    #[inline]
     fn parent_ctxt(&self, ctxt: SyntaxContext) -> SyntaxContext {
         self.syntax_context_data[ctxt.0 as usize].parent
     }
@@ -718,11 +726,13 @@ impl SyntaxContext {
         SyntaxContext(raw as u32)
     }
 
+    #[inline]
     fn from_usize(raw: usize) -> SyntaxContext {
         SyntaxContext(u32::try_from(raw).unwrap())
     }
 
     /// Extend a syntax context with a given expansion and transparency.
+    #[inline]
     pub fn apply_mark(self, expn_id: ExpnId, transparency: Transparency) -> SyntaxContext {
         HygieneData::with(|data| data.apply_mark(self, expn_id, transparency))
     }
@@ -743,10 +753,12 @@ impl SyntaxContext {
     /// of g (call it g1), calling remove_mark will result in the SyntaxContext for the
     /// invocation of f that created g1.
     /// Returns the mark that was removed.
+    #[inline]
     pub fn remove_mark(&mut self) -> ExpnId {
         HygieneData::with(|data| data.remove_mark(self).0)
     }
 
+    #[inline]
     pub fn marks(self) -> Vec<(ExpnId, Transparency)> {
         HygieneData::with(|data| data.marks(self))
     }
@@ -776,11 +788,13 @@ impl SyntaxContext {
     /// ```
     /// This returns the expansion whose definition scope we use to privacy check the resolution,
     /// or `None` if we privacy check as usual (i.e., not w.r.t. a macro definition scope).
+    #[inline]
     pub fn adjust(&mut self, expn_id: ExpnId) -> Option<ExpnId> {
         HygieneData::with(|data| data.adjust(self, expn_id))
     }
 
     /// Like `SyntaxContext::adjust`, but also normalizes `self` to macros 2.0.
+    #[inline]
     pub(crate) fn normalize_to_macros_2_0_and_adjust(&mut self, expn_id: ExpnId) -> Option<ExpnId> {
         HygieneData::with(|data| {
             *self = data.normalize_to_macros_2_0(*self);
@@ -901,10 +915,12 @@ impl SyntaxContext {
         HygieneData::with(|data| data.outer_mark(self))
     }
 
+    #[inline]
     pub(crate) fn dollar_crate_name(self) -> Symbol {
         HygieneData::with(|data| data.syntax_context_data[self.0 as usize].dollar_crate_name)
     }
 
+    #[inline]
     pub fn edition(self) -> Edition {
         HygieneData::with(|data| data.expn_data(data.outer_expn(self)).edition)
     }
diff --git a/compiler/rustc_span/src/lib.rs b/compiler/rustc_span/src/lib.rs
index 9b0e009b2cd..dbc67da37b5 100644
--- a/compiler/rustc_span/src/lib.rs
+++ b/compiler/rustc_span/src/lib.rs
@@ -167,6 +167,7 @@ where
     }
 }
 
+#[inline]
 pub fn with_session_globals<R, F>(f: F) -> R
 where
     F: FnOnce(&SessionGlobals) -> R,
diff --git a/compiler/rustc_target/Cargo.toml b/compiler/rustc_target/Cargo.toml
index 0121c752dbd..56932c24922 100644
--- a/compiler/rustc_target/Cargo.toml
+++ b/compiler/rustc_target/Cargo.toml
@@ -12,7 +12,10 @@ rustc_fs_util = { path = "../rustc_fs_util" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_span = { path = "../rustc_span" }
+serde = "1.0.219"
+serde_derive = "1.0.219"
 serde_json = "1.0.59"
+serde_path_to_error = "0.1.17"
 tracing = "0.1"
 # tidy-alphabetical-end
 
diff --git a/compiler/rustc_target/src/json.rs b/compiler/rustc_target/src/json.rs
index 4fcc477921b..896609bdbe3 100644
--- a/compiler/rustc_target/src/json.rs
+++ b/compiler/rustc_target/src/json.rs
@@ -114,3 +114,18 @@ impl ToJson for rustc_abi::CanonAbi {
         self.to_string().to_json()
     }
 }
+
+macro_rules! serde_deserialize_from_str {
+    ($ty:ty) => {
+        impl<'de> serde::Deserialize<'de> for $ty {
+            fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+            where
+                D: serde::Deserializer<'de>,
+            {
+                let s = String::deserialize(deserializer)?;
+                FromStr::from_str(&s).map_err(serde::de::Error::custom)
+            }
+        }
+    };
+}
+pub(crate) use serde_deserialize_from_str;
diff --git a/compiler/rustc_target/src/spec/json.rs b/compiler/rustc_target/src/spec/json.rs
index 6c716f87125..d27c1929aef 100644
--- a/compiler/rustc_target/src/spec/json.rs
+++ b/compiler/rustc_target/src/spec/json.rs
@@ -1,60 +1,47 @@
-use std::borrow::Cow;
 use std::collections::BTreeMap;
 use std::str::FromStr;
 
-use rustc_abi::{Align, AlignFromBytesError, ExternAbi};
-use serde_json::Value;
+use rustc_abi::{Align, AlignFromBytesError};
 
-use super::{Target, TargetKind, TargetOptions, TargetWarnings};
+use super::crt_objects::CrtObjects;
+use super::{
+    BinaryFormat, CodeModel, DebuginfoKind, FloatAbi, FramePointer, LinkArgsCli,
+    LinkSelfContainedComponents, LinkSelfContainedDefault, LinkerFlavorCli, LldFlavor,
+    MergeFunctions, PanicStrategy, RelocModel, RelroLevel, RustcAbi, SanitizerSet,
+    SmallDataThresholdSupport, SplitDebuginfo, StackProbeType, StaticCow, SymbolVisibility, Target,
+    TargetKind, TargetOptions, TargetWarnings, TlsModel,
+};
 use crate::json::{Json, ToJson};
 use crate::spec::AbiMap;
 
 impl Target {
     /// Loads a target descriptor from a JSON object.
-    pub fn from_json(obj: Json) -> Result<(Target, TargetWarnings), String> {
-        // While ugly, this code must remain this way to retain
-        // compatibility with existing JSON fields and the internal
-        // expected naming of the Target and TargetOptions structs.
-        // To ensure compatibility is retained, the built-in targets
-        // are round-tripped through this code to catch cases where
-        // the JSON parser is not updated to match the structs.
-
-        let mut obj = match obj {
-            Value::Object(obj) => obj,
-            _ => return Err("Expected JSON object for target")?,
-        };
+    pub fn from_json(json: &str) -> Result<(Target, TargetWarnings), String> {
+        let json_deserializer = &mut serde_json::Deserializer::from_str(json);
 
-        let mut get_req_field = |name: &str| {
-            obj.remove(name)
-                .and_then(|j| j.as_str().map(str::to_string))
-                .ok_or_else(|| format!("Field {name} in target specification is required"))
-        };
+        let json: TargetSpecJson =
+            serde_path_to_error::deserialize(json_deserializer).map_err(|err| err.to_string())?;
 
         let mut base = Target {
-            llvm_target: get_req_field("llvm-target")?.into(),
+            llvm_target: json.llvm_target,
             metadata: Default::default(),
-            pointer_width: get_req_field("target-pointer-width")?
-                .parse::<u32>()
-                .map_err(|_| "target-pointer-width must be an integer".to_string())?,
-            data_layout: get_req_field("data-layout")?.into(),
-            arch: get_req_field("arch")?.into(),
+            pointer_width: json
+                .target_pointer_width
+                .parse()
+                .map_err(|err| format!("invalid target-pointer-width: {err}"))?,
+            data_layout: json.data_layout,
+            arch: json.arch,
             options: Default::default(),
         };
 
         // FIXME: This doesn't properly validate anything and just ignores the data if it's invalid.
         // That's okay for now, the only use of this is when generating docs, which we don't do for
         // custom targets.
-        if let Some(Json::Object(mut metadata)) = obj.remove("metadata") {
-            base.metadata.description = metadata
-                .remove("description")
-                .and_then(|desc| desc.as_str().map(|desc| desc.to_owned().into()));
-            base.metadata.tier = metadata
-                .remove("tier")
-                .and_then(|tier| tier.as_u64())
-                .filter(|tier| (1..=3).contains(tier));
-            base.metadata.host_tools =
-                metadata.remove("host_tools").and_then(|host| host.as_bool());
-            base.metadata.std = metadata.remove("std").and_then(|host| host.as_bool());
+        if let Some(metadata) = json.metadata {
+            base.metadata.description = metadata.description;
+            base.metadata.tier = metadata.tier.filter(|tier| (1..=3).contains(tier));
+            base.metadata.host_tools = metadata.host_tools;
+            base.metadata.std = metadata.std;
         }
 
         let alignment_error = |field_name: &str, error: AlignFromBytesError| -> String {
@@ -65,640 +52,188 @@ impl Target {
             format!("`{}` bits is not a valid value for {field_name}: {msg}", error.align() * 8)
         };
 
-        let mut incorrect_type = vec![];
-
-        macro_rules! key {
-            ($key_name:ident) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                if let Some(s) = obj.remove(&name).and_then(|s| s.as_str().map(str::to_string).map(Cow::from)) {
-                    base.$key_name = s;
-                }
-            } );
-            ($key_name:ident = $json_name:expr) => ( {
-                let name = $json_name;
-                if let Some(s) = obj.remove(name).and_then(|s| s.as_str().map(str::to_string).map(Cow::from)) {
-                    base.$key_name = s;
-                }
-            } );
-            ($key_name:ident = $json_name:expr, u64 as $int_ty:ty) => ( {
-                let name = $json_name;
-                if let Some(s) = obj.remove(name).and_then(|b| b.as_u64()) {
-                    base.$key_name = s as $int_ty;
-                }
-            } );
-            ($key_name:ident, bool) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                if let Some(s) = obj.remove(&name).and_then(|b| b.as_bool()) {
-                    base.$key_name = s;
-                }
-            } );
-            ($key_name:ident = $json_name:expr, bool) => ( {
-                let name = $json_name;
-                if let Some(s) = obj.remove(name).and_then(|b| b.as_bool()) {
-                    base.$key_name = s;
-                }
-            } );
-            ($key_name:ident, u32) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                if let Some(s) = obj.remove(&name).and_then(|b| b.as_u64()) {
-                    if s < 1 || s > 5 {
-                        return Err("Not a valid DWARF version number".into());
-                    }
-                    base.$key_name = s as u32;
-                }
-            } );
-            ($key_name:ident, Option<bool>) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                if let Some(s) = obj.remove(&name).and_then(|b| b.as_bool()) {
-                    base.$key_name = Some(s);
-                }
-            } );
-            ($key_name:ident, Option<u64>) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                if let Some(s) = obj.remove(&name).and_then(|b| b.as_u64()) {
-                    base.$key_name = Some(s);
-                }
-            } );
-            ($key_name:ident, Option<StaticCow<str>>) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                if let Some(s) = obj.remove(&name).and_then(|b| Some(b.as_str()?.to_string())) {
-                    base.$key_name = Some(s.into());
-                }
-            } );
-            ($key_name:ident, Option<Align>) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                if let Some(b) = obj.remove(&name).and_then(|b| b.as_u64()) {
-                    match Align::from_bits(b) {
-                        Ok(align) => base.$key_name = Some(align),
-                        Err(e) => return Err(alignment_error(&name, e)),
-                    }
-                }
-            } );
-            ($key_name:ident, BinaryFormat) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                obj.remove(&name).and_then(|f| f.as_str().and_then(|s| {
-                    match s.parse::<super::BinaryFormat>() {
-                        Ok(binary_format) => base.$key_name = binary_format,
-                        _ => return Some(Err(format!(
-                            "'{s}' is not a valid value for binary_format. \
-                            Use 'coff', 'elf', 'mach-o', 'wasm' or 'xcoff' "
-                        ))),
-                    }
-                    Some(Ok(()))
-                })).unwrap_or(Ok(()))
-            } );
-            ($key_name:ident, MergeFunctions) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                obj.remove(&name).and_then(|o| o.as_str().and_then(|s| {
-                    match s.parse::<super::MergeFunctions>() {
-                        Ok(mergefunc) => base.$key_name = mergefunc,
-                        _ => return Some(Err(format!("'{}' is not a valid value for \
-                                                      merge-functions. Use 'disabled', \
-                                                      'trampolines', or 'aliases'.",
-                                                      s))),
-                    }
-                    Some(Ok(()))
-                })).unwrap_or(Ok(()))
-            } );
-            ($key_name:ident, FloatAbi) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                obj.remove(&name).and_then(|o| o.as_str().and_then(|s| {
-                    match s.parse::<super::FloatAbi>() {
-                        Ok(float_abi) => base.$key_name = Some(float_abi),
-                        _ => return Some(Err(format!("'{}' is not a valid value for \
-                                                      llvm-floatabi. Use 'soft' or 'hard'.",
-                                                      s))),
-                    }
-                    Some(Ok(()))
-                })).unwrap_or(Ok(()))
-            } );
-            ($key_name:ident, RustcAbi) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                obj.remove(&name).and_then(|o| o.as_str().and_then(|s| {
-                    match s.parse::<super::RustcAbi>() {
-                        Ok(rustc_abi) => base.$key_name = Some(rustc_abi),
-                        _ => return Some(Err(format!(
-                            "'{s}' is not a valid value for rustc-abi. \
-                            Use 'x86-softfloat' or leave the field unset."
-                        ))),
-                    }
-                    Some(Ok(()))
-                })).unwrap_or(Ok(()))
-            } );
-            ($key_name:ident, RelocModel) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                obj.remove(&name).and_then(|o| o.as_str().and_then(|s| {
-                    match s.parse::<super::RelocModel>() {
-                        Ok(relocation_model) => base.$key_name = relocation_model,
-                        _ => return Some(Err(format!("'{}' is not a valid relocation model. \
-                                                      Run `rustc --print relocation-models` to \
-                                                      see the list of supported values.", s))),
-                    }
-                    Some(Ok(()))
-                })).unwrap_or(Ok(()))
-            } );
-            ($key_name:ident, CodeModel) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                obj.remove(&name).and_then(|o| o.as_str().and_then(|s| {
-                    match s.parse::<super::CodeModel>() {
-                        Ok(code_model) => base.$key_name = Some(code_model),
-                        _ => return Some(Err(format!("'{}' is not a valid code model. \
-                                                      Run `rustc --print code-models` to \
-                                                      see the list of supported values.", s))),
-                    }
-                    Some(Ok(()))
-                })).unwrap_or(Ok(()))
-            } );
-            ($key_name:ident, TlsModel) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                obj.remove(&name).and_then(|o| o.as_str().and_then(|s| {
-                    match s.parse::<super::TlsModel>() {
-                        Ok(tls_model) => base.$key_name = tls_model,
-                        _ => return Some(Err(format!("'{}' is not a valid TLS model. \
-                                                      Run `rustc --print tls-models` to \
-                                                      see the list of supported values.", s))),
-                    }
-                    Some(Ok(()))
-                })).unwrap_or(Ok(()))
-            } );
-            ($key_name:ident, SmallDataThresholdSupport) => ( {
-                obj.remove("small-data-threshold-support").and_then(|o| o.as_str().and_then(|s| {
-                    match s.parse::<super::SmallDataThresholdSupport>() {
-                        Ok(support) => base.small_data_threshold_support = support,
-                        _ => return Some(Err(format!("'{s}' is not a valid value for small-data-threshold-support."))),
-                    }
-                    Some(Ok(()))
-                })).unwrap_or(Ok(()))
-            } );
-            ($key_name:ident, PanicStrategy) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                obj.remove(&name).and_then(|o| o.as_str().and_then(|s| {
-                    match s {
-                        "unwind" => base.$key_name = super::PanicStrategy::Unwind,
-                        "abort" => base.$key_name = super::PanicStrategy::Abort,
-                        _ => return Some(Err(format!("'{}' is not a valid value for \
-                                                      panic-strategy. Use 'unwind' or 'abort'.",
-                                                     s))),
-                }
-                Some(Ok(()))
-            })).unwrap_or(Ok(()))
-            } );
-            ($key_name:ident, RelroLevel) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                obj.remove(&name).and_then(|o| o.as_str().and_then(|s| {
-                    match s.parse::<super::RelroLevel>() {
-                        Ok(level) => base.$key_name = level,
-                        _ => return Some(Err(format!("'{}' is not a valid value for \
-                                                      relro-level. Use 'full', 'partial, or 'off'.",
-                                                      s))),
-                    }
-                    Some(Ok(()))
-                })).unwrap_or(Ok(()))
-            } );
-            ($key_name:ident, Option<SymbolVisibility>) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                obj.remove(&name).and_then(|o| o.as_str().and_then(|s| {
-                    match s.parse::<super::SymbolVisibility>() {
-                        Ok(level) => base.$key_name = Some(level),
-                        _ => return Some(Err(format!("'{}' is not a valid value for \
-                                                      symbol-visibility. Use 'hidden', 'protected, or 'interposable'.",
-                                                      s))),
-                    }
-                    Some(Ok(()))
-                })).unwrap_or(Ok(()))
-            } );
-            ($key_name:ident, DebuginfoKind) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                obj.remove(&name).and_then(|o| o.as_str().and_then(|s| {
-                    match s.parse::<super::DebuginfoKind>() {
-                        Ok(level) => base.$key_name = level,
-                        _ => return Some(Err(
-                            format!("'{s}' is not a valid value for debuginfo-kind. Use 'dwarf', \
-                                  'dwarf-dsym' or 'pdb'.")
-                        )),
-                    }
-                    Some(Ok(()))
-                })).unwrap_or(Ok(()))
-            } );
-            ($key_name:ident, SplitDebuginfo) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                obj.remove(&name).and_then(|o| o.as_str().and_then(|s| {
-                    match s.parse::<super::SplitDebuginfo>() {
-                        Ok(level) => base.$key_name = level,
-                        _ => return Some(Err(format!("'{}' is not a valid value for \
-                                                      split-debuginfo. Use 'off' or 'dsymutil'.",
-                                                      s))),
-                    }
-                    Some(Ok(()))
-                })).unwrap_or(Ok(()))
-            } );
-            ($key_name:ident, list) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                if let Some(j) = obj.remove(&name) {
-                    if let Some(v) = j.as_array() {
-                        base.$key_name = v.iter()
-                            .map(|a| a.as_str().unwrap().to_string().into())
-                            .collect();
-                    } else {
-                        incorrect_type.push(name)
-                    }
-                }
-            } );
-            ($key_name:ident, opt_list) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                if let Some(j) = obj.remove(&name) {
-                    if let Some(v) = j.as_array() {
-                        base.$key_name = Some(v.iter()
-                            .map(|a| a.as_str().unwrap().to_string().into())
-                            .collect());
-                    } else {
-                        incorrect_type.push(name)
-                    }
+        macro_rules! forward {
+            ($name:ident) => {
+                if let Some($name) = json.$name {
+                    base.$name = $name;
                 }
-            } );
-            ($key_name:ident, fallible_list) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                obj.remove(&name).and_then(|j| {
-                    if let Some(v) = j.as_array() {
-                        match v.iter().map(|a| FromStr::from_str(a.as_str().unwrap())).collect() {
-                            Ok(l) => { base.$key_name = l },
-                            // FIXME: `fallible_list` can't re-use the `key!` macro for list
-                            // elements and the error messages from that macro, so it has a bad
-                            // generic message instead
-                            Err(_) => return Some(Err(
-                                format!("`{:?}` is not a valid value for `{}`", j, name)
-                            )),
-                        }
-                    } else {
-                        incorrect_type.push(name)
-                    }
-                    Some(Ok(()))
-                }).unwrap_or(Ok(()))
-            } );
-            ($key_name:ident, optional) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                if let Some(o) = obj.remove(&name) {
-                    base.$key_name = o
-                        .as_str()
-                        .map(|s| s.to_string().into());
-                }
-            } );
-            ($key_name:ident = $json_name:expr, LldFlavor) => ( {
-                let name = $json_name;
-                obj.remove(name).and_then(|o| o.as_str().and_then(|s| {
-                    if let Some(flavor) = super::LldFlavor::from_str(&s) {
-                        base.$key_name = flavor;
-                    } else {
-                        return Some(Err(format!(
-                            "'{}' is not a valid value for lld-flavor. \
-                             Use 'darwin', 'gnu', 'link' or 'wasm'.",
-                            s)))
-                    }
-                    Some(Ok(()))
-                })).unwrap_or(Ok(()))
-            } );
-            ($key_name:ident = $json_name:expr, LinkerFlavorCli) => ( {
-                let name = $json_name;
-                obj.remove(name).and_then(|o| o.as_str().and_then(|s| {
-                    match super::LinkerFlavorCli::from_str(s) {
-                        Some(linker_flavor) => base.$key_name = linker_flavor,
-                        _ => return Some(Err(format!("'{}' is not a valid value for linker-flavor. \
-                                                      Use {}", s, super::LinkerFlavorCli::one_of()))),
-                    }
-                    Some(Ok(()))
-                })).unwrap_or(Ok(()))
-            } );
-            ($key_name:ident, StackProbeType) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                obj.remove(&name).and_then(|o| match super::StackProbeType::from_json(&o) {
-                    Ok(v) => {
-                        base.$key_name = v;
-                        Some(Ok(()))
-                    },
-                    Err(s) => Some(Err(
-                        format!("`{:?}` is not a valid value for `{}`: {}", o, name, s)
-                    )),
-                }).unwrap_or(Ok(()))
-            } );
-            ($key_name:ident, SanitizerSet) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                if let Some(o) = obj.remove(&name) {
-                    if let Some(a) = o.as_array() {
-                        for s in a {
-                            use super::SanitizerSet;
-                            base.$key_name |= match s.as_str() {
-                                Some("address") => SanitizerSet::ADDRESS,
-                                Some("cfi") => SanitizerSet::CFI,
-                                Some("dataflow") => SanitizerSet::DATAFLOW,
-                                Some("kcfi") => SanitizerSet::KCFI,
-                                Some("kernel-address") => SanitizerSet::KERNELADDRESS,
-                                Some("leak") => SanitizerSet::LEAK,
-                                Some("memory") => SanitizerSet::MEMORY,
-                                Some("memtag") => SanitizerSet::MEMTAG,
-                                Some("safestack") => SanitizerSet::SAFESTACK,
-                                Some("shadow-call-stack") => SanitizerSet::SHADOWCALLSTACK,
-                                Some("thread") => SanitizerSet::THREAD,
-                                Some("hwaddress") => SanitizerSet::HWADDRESS,
-                                Some(s) => return Err(format!("unknown sanitizer {}", s)),
-                                _ => return Err(format!("not a string: {:?}", s)),
-                            };
-                        }
-                    } else {
-                        incorrect_type.push(name)
-                    }
-                }
-                Ok::<(), String>(())
-            } );
-            ($key_name:ident, link_self_contained_components) => ( {
-                // Skeleton of what needs to be parsed:
-                //
-                // ```
-                // $name: {
-                //     "components": [
-                //         <array of strings>
-                //     ]
-                // }
-                // ```
-                let name = (stringify!($key_name)).replace("_", "-");
-                if let Some(o) = obj.remove(&name) {
-                    if let Some(o) = o.as_object() {
-                        let component_array = o.get("components")
-                            .ok_or_else(|| format!("{name}: expected a \
-                                JSON object with a `components` field."))?;
-                        let component_array = component_array.as_array()
-                            .ok_or_else(|| format!("{name}.components: expected a JSON array"))?;
-                        let mut components = super::LinkSelfContainedComponents::empty();
-                        for s in component_array {
-                            components |= match s.as_str() {
-                                Some(s) => {
-                                    super::LinkSelfContainedComponents::from_str(s)
-                                        .ok_or_else(|| format!("unknown \
-                                        `-Clink-self-contained` component: {s}"))?
-                                },
-                                _ => return Err(format!("not a string: {:?}", s)),
-                            };
-                        }
-                        base.$key_name = super::LinkSelfContainedDefault::WithComponents(components);
-                    } else {
-                        incorrect_type.push(name)
-                    }
-                }
-                Ok::<(), String>(())
-            } );
-            ($key_name:ident = $json_name:expr, link_self_contained_backwards_compatible) => ( {
-                let name = $json_name;
-                obj.remove(name).and_then(|o| o.as_str().and_then(|s| {
-                    match s.parse::<super::LinkSelfContainedDefault>() {
-                        Ok(lsc_default) => base.$key_name = lsc_default,
-                        _ => return Some(Err(format!("'{}' is not a valid `-Clink-self-contained` default. \
-                                                      Use 'false', 'true', 'musl' or 'mingw'", s))),
-                    }
-                    Some(Ok(()))
-                })).unwrap_or(Ok(()))
-            } );
-            ($key_name:ident = $json_name:expr, link_objects) => ( {
-                let name = $json_name;
-                if let Some(val) = obj.remove(name) {
-                    let obj = val.as_object().ok_or_else(|| format!("{}: expected a \
-                        JSON object with fields per CRT object kind.", name))?;
-                    let mut args = super::CrtObjects::new();
-                    for (k, v) in obj {
-                        let kind = super::LinkOutputKind::from_str(&k).ok_or_else(|| {
-                            format!("{}: '{}' is not a valid value for CRT object kind. \
-                                     Use '(dynamic,static)-(nopic,pic)-exe' or \
-                                     '(dynamic,static)-dylib' or 'wasi-reactor-exe'", name, k)
-                        })?;
-
-                        let v = v.as_array().ok_or_else(||
-                            format!("{}.{}: expected a JSON array", name, k)
-                        )?.iter().enumerate()
-                            .map(|(i,s)| {
-                                let s = s.as_str().ok_or_else(||
-                                    format!("{}.{}[{}]: expected a JSON string", name, k, i))?;
-                                Ok(s.to_string().into())
-                            })
-                            .collect::<Result<Vec<_>, String>>()?;
-
-                        args.insert(kind, v);
-                    }
-                    base.$key_name = args;
-                }
-            } );
-            ($key_name:ident = $json_name:expr, link_args) => ( {
-                let name = $json_name;
-                if let Some(val) = obj.remove(name) {
-                    let obj = val.as_object().ok_or_else(|| format!("{}: expected a \
-                        JSON object with fields per linker-flavor.", name))?;
-                    let mut args = super::LinkArgsCli::new();
-                    for (k, v) in obj {
-                        let flavor = super::LinkerFlavorCli::from_str(&k).ok_or_else(|| {
-                            format!("{}: '{}' is not a valid value for linker-flavor. \
-                                     Use 'em', 'gcc', 'ld' or 'msvc'", name, k)
-                        })?;
-
-                        let v = v.as_array().ok_or_else(||
-                            format!("{}.{}: expected a JSON array", name, k)
-                        )?.iter().enumerate()
-                            .map(|(i,s)| {
-                                let s = s.as_str().ok_or_else(||
-                                    format!("{}.{}[{}]: expected a JSON string", name, k, i))?;
-                                Ok(s.to_string().into())
-                            })
-                            .collect::<Result<Vec<_>, String>>()?;
-
-                        args.insert(flavor, v);
-                    }
-                    base.$key_name = args;
-                }
-            } );
-            ($key_name:ident, env) => ( {
-                let name = (stringify!($key_name)).replace("_", "-");
-                if let Some(o) = obj.remove(&name) {
-                    if let Some(a) = o.as_array() {
-                        for o in a {
-                            if let Some(s) = o.as_str() {
-                                if let [k, v] = *s.split('=').collect::<Vec<_>>() {
-                                    base.$key_name
-                                        .to_mut()
-                                        .push((k.to_string().into(), v.to_string().into()))
-                                }
-                            }
-                        }
-                    } else {
-                        incorrect_type.push(name)
-                    }
+            };
+        }
+        macro_rules! forward_opt {
+            ($name:ident) => {
+                if let Some($name) = json.$name {
+                    base.$name = Some($name);
                 }
-            } );
-            ($key_name:ident, target_families) => ( {
-                if let Some(value) = obj.remove("target-family") {
-                    if let Some(v) = value.as_array() {
-                        base.$key_name = v.iter()
-                            .map(|a| a.as_str().unwrap().to_string().into())
-                            .collect();
-                    } else if let Some(v) = value.as_str() {
-                        base.$key_name = vec![v.to_string().into()].into();
-                    }
+            };
+        }
+
+        if let Some(target_endian) = json.target_endian {
+            base.endian = target_endian.0;
+        }
+
+        forward!(frame_pointer);
+        forward!(c_int_width);
+        forward_opt!(c_enum_min_bits); // if None, matches c_int_width
+        forward!(os);
+        forward!(env);
+        forward!(abi);
+        forward!(vendor);
+        forward_opt!(linker);
+        forward!(linker_flavor_json);
+        forward!(lld_flavor_json);
+        forward!(linker_is_gnu_json);
+        forward!(pre_link_objects);
+        forward!(post_link_objects);
+        forward!(pre_link_objects_self_contained);
+        forward!(post_link_objects_self_contained);
+
+        // Deserializes the backwards-compatible variants of `-Clink-self-contained`
+        if let Some(link_self_contained) = json.link_self_contained_backwards_compatible {
+            base.link_self_contained = link_self_contained;
+        }
+        // Deserializes the components variant of `-Clink-self-contained`
+        if let Some(link_self_contained) = json.link_self_contained {
+            let components = link_self_contained
+                .components
+                .into_iter()
+                .fold(LinkSelfContainedComponents::empty(), |a, b| a | b);
+            base.link_self_contained = LinkSelfContainedDefault::WithComponents(components);
+        }
+
+        forward!(pre_link_args_json);
+        forward!(late_link_args_json);
+        forward!(late_link_args_dynamic_json);
+        forward!(late_link_args_static_json);
+        forward!(post_link_args_json);
+        forward_opt!(link_script);
+
+        if let Some(link_env) = json.link_env {
+            for s in link_env {
+                if let [k, v] = *s.split('=').collect::<Vec<_>>() {
+                    base.link_env.to_mut().push((k.to_string().into(), v.to_string().into()))
+                } else {
+                    return Err(format!("link-env value '{s}' must be of the pattern 'KEY=VALUE'"));
                 }
-            } );
+            }
         }
 
-        if let Some(j) = obj.remove("target-endian") {
-            if let Some(s) = j.as_str() {
-                base.endian = s.parse()?;
-            } else {
-                incorrect_type.push("target-endian".into())
+        forward!(link_env_remove);
+        forward!(asm_args);
+        forward!(cpu);
+        forward!(need_explicit_cpu);
+        forward!(features);
+        forward!(dynamic_linking);
+        forward_opt!(direct_access_external_data);
+        forward!(dll_tls_export);
+        forward!(only_cdylib);
+        forward!(executables);
+        forward!(relocation_model);
+        forward_opt!(code_model);
+        forward!(tls_model);
+        forward!(disable_redzone);
+        forward!(function_sections);
+        forward!(dll_prefix);
+        forward!(dll_suffix);
+        forward!(exe_suffix);
+        forward!(staticlib_prefix);
+        forward!(staticlib_suffix);
+
+        if let Some(target_family) = json.target_family {
+            match target_family {
+                TargetFamiliesJson::Array(families) => base.families = families,
+                TargetFamiliesJson::String(family) => base.families = vec![family].into(),
             }
         }
 
-        if let Some(fp) = obj.remove("frame-pointer") {
-            if let Some(s) = fp.as_str() {
-                base.frame_pointer = s
-                    .parse()
-                    .map_err(|()| format!("'{s}' is not a valid value for frame-pointer"))?;
-            } else {
-                incorrect_type.push("frame-pointer".into())
+        forward!(abi_return_struct_as_int);
+        forward!(is_like_aix);
+        forward!(is_like_darwin);
+        forward!(is_like_solaris);
+        forward!(is_like_windows);
+        forward!(is_like_msvc);
+        forward!(is_like_wasm);
+        forward!(is_like_android);
+        forward!(binary_format);
+        forward!(default_dwarf_version);
+        forward!(allows_weak_linkage);
+        forward!(has_rpath);
+        forward!(no_default_libraries);
+        forward!(position_independent_executables);
+        forward!(static_position_independent_executables);
+        forward!(plt_by_default);
+        forward!(relro_level);
+        forward!(archive_format);
+        forward!(allow_asm);
+        forward!(main_needs_argc_argv);
+        forward!(has_thread_local);
+        forward!(obj_is_bitcode);
+        forward!(bitcode_llvm_cmdline);
+        forward_opt!(max_atomic_width);
+        forward_opt!(min_atomic_width);
+        forward!(atomic_cas);
+        forward!(panic_strategy);
+        forward!(crt_static_allows_dylibs);
+        forward!(crt_static_default);
+        forward!(crt_static_respected);
+        forward!(stack_probes);
+
+        if let Some(min_global_align) = json.min_global_align {
+            match Align::from_bits(min_global_align) {
+                Ok(align) => base.min_global_align = Some(align),
+                Err(e) => return Err(alignment_error("min-global-align", e)),
             }
         }
 
-        key!(c_int_width = "target-c-int-width", u64 as u16);
-        key!(c_enum_min_bits, Option<u64>); // if None, matches c_int_width
-        key!(os);
-        key!(env);
-        key!(abi);
-        key!(vendor);
-        key!(linker, optional);
-        key!(linker_flavor_json = "linker-flavor", LinkerFlavorCli)?;
-        key!(lld_flavor_json = "lld-flavor", LldFlavor)?;
-        key!(linker_is_gnu_json = "linker-is-gnu", bool);
-        key!(pre_link_objects = "pre-link-objects", link_objects);
-        key!(post_link_objects = "post-link-objects", link_objects);
-        key!(pre_link_objects_self_contained = "pre-link-objects-fallback", link_objects);
-        key!(post_link_objects_self_contained = "post-link-objects-fallback", link_objects);
-        // Deserializes the backwards-compatible variants of `-Clink-self-contained`
-        key!(
-            link_self_contained = "crt-objects-fallback",
-            link_self_contained_backwards_compatible
-        )?;
-        // Deserializes the components variant of `-Clink-self-contained`
-        key!(link_self_contained, link_self_contained_components)?;
-        key!(pre_link_args_json = "pre-link-args", link_args);
-        key!(late_link_args_json = "late-link-args", link_args);
-        key!(late_link_args_dynamic_json = "late-link-args-dynamic", link_args);
-        key!(late_link_args_static_json = "late-link-args-static", link_args);
-        key!(post_link_args_json = "post-link-args", link_args);
-        key!(link_script, optional);
-        key!(link_env, env);
-        key!(link_env_remove, list);
-        key!(asm_args, list);
-        key!(cpu);
-        key!(need_explicit_cpu, bool);
-        key!(features);
-        key!(dynamic_linking, bool);
-        key!(direct_access_external_data, Option<bool>);
-        key!(dll_tls_export, bool);
-        key!(only_cdylib, bool);
-        key!(executables, bool);
-        key!(relocation_model, RelocModel)?;
-        key!(code_model, CodeModel)?;
-        key!(tls_model, TlsModel)?;
-        key!(disable_redzone, bool);
-        key!(function_sections, bool);
-        key!(dll_prefix);
-        key!(dll_suffix);
-        key!(exe_suffix);
-        key!(staticlib_prefix);
-        key!(staticlib_suffix);
-        key!(families, target_families);
-        key!(abi_return_struct_as_int, bool);
-        key!(is_like_aix, bool);
-        key!(is_like_darwin, bool);
-        key!(is_like_solaris, bool);
-        key!(is_like_windows, bool);
-        key!(is_like_msvc, bool);
-        key!(is_like_wasm, bool);
-        key!(is_like_android, bool);
-        key!(binary_format, BinaryFormat)?;
-        key!(default_dwarf_version, u32);
-        key!(allows_weak_linkage, bool);
-        key!(has_rpath, bool);
-        key!(no_default_libraries, bool);
-        key!(position_independent_executables, bool);
-        key!(static_position_independent_executables, bool);
-        key!(plt_by_default, bool);
-        key!(relro_level, RelroLevel)?;
-        key!(archive_format);
-        key!(allow_asm, bool);
-        key!(main_needs_argc_argv, bool);
-        key!(has_thread_local, bool);
-        key!(obj_is_bitcode, bool);
-        key!(bitcode_llvm_cmdline);
-        key!(max_atomic_width, Option<u64>);
-        key!(min_atomic_width, Option<u64>);
-        key!(atomic_cas, bool);
-        key!(panic_strategy, PanicStrategy)?;
-        key!(crt_static_allows_dylibs, bool);
-        key!(crt_static_default, bool);
-        key!(crt_static_respected, bool);
-        key!(stack_probes, StackProbeType)?;
-        key!(min_global_align, Option<Align>);
-        key!(default_codegen_units, Option<u64>);
-        key!(default_codegen_backend, Option<StaticCow<str>>);
-        key!(trap_unreachable, bool);
-        key!(requires_lto, bool);
-        key!(singlethread, bool);
-        key!(no_builtins, bool);
-        key!(default_visibility, Option<SymbolVisibility>)?;
-        key!(emit_debug_gdb_scripts, bool);
-        key!(requires_uwtable, bool);
-        key!(default_uwtable, bool);
-        key!(simd_types_indirect, bool);
-        key!(limit_rdylib_exports, bool);
-        key!(override_export_symbols, opt_list);
-        key!(merge_functions, MergeFunctions)?;
-        key!(mcount = "target-mcount");
-        key!(llvm_mcount_intrinsic, optional);
-        key!(llvm_abiname);
-        key!(llvm_floatabi, FloatAbi)?;
-        key!(rustc_abi, RustcAbi)?;
-        key!(relax_elf_relocations, bool);
-        key!(llvm_args, list);
-        key!(use_ctors_section, bool);
-        key!(eh_frame_header, bool);
-        key!(has_thumb_interworking, bool);
-        key!(debuginfo_kind, DebuginfoKind)?;
-        key!(split_debuginfo, SplitDebuginfo)?;
-        key!(supported_split_debuginfo, fallible_list)?;
-        key!(supported_sanitizers, SanitizerSet)?;
-        key!(generate_arange_section, bool);
-        key!(supports_stack_protector, bool);
-        key!(small_data_threshold_support, SmallDataThresholdSupport)?;
-        key!(entry_name);
-        key!(supports_xray, bool);
+        forward_opt!(default_codegen_units);
+        forward_opt!(default_codegen_backend);
+        forward!(trap_unreachable);
+        forward!(requires_lto);
+        forward!(singlethread);
+        forward!(no_builtins);
+        forward_opt!(default_visibility);
+        forward!(emit_debug_gdb_scripts);
+        forward!(requires_uwtable);
+        forward!(default_uwtable);
+        forward!(simd_types_indirect);
+        forward!(limit_rdylib_exports);
+        forward_opt!(override_export_symbols);
+        forward!(merge_functions);
+        forward!(mcount);
+        forward_opt!(llvm_mcount_intrinsic);
+        forward!(llvm_abiname);
+        forward_opt!(llvm_floatabi);
+        forward_opt!(rustc_abi);
+        forward!(relax_elf_relocations);
+        forward!(llvm_args);
+        forward!(use_ctors_section);
+        forward!(eh_frame_header);
+        forward!(has_thumb_interworking);
+        forward!(debuginfo_kind);
+        forward!(split_debuginfo);
+        forward!(supported_split_debuginfo);
+
+        if let Some(supported_sanitizers) = json.supported_sanitizers {
+            base.supported_sanitizers =
+                supported_sanitizers.into_iter().fold(SanitizerSet::empty(), |a, b| a | b);
+        }
+
+        forward!(generate_arange_section);
+        forward!(supports_stack_protector);
+        forward!(small_data_threshold_support);
+        forward!(entry_name);
+        forward!(supports_xray);
 
         // we're going to run `update_from_cli`, but that won't change the target's AbiMap
         // FIXME: better factor the Target definition so we enforce this on a type level
         let abi_map = AbiMap::from_target(&base);
-
-        if let Some(abi_str) = obj.remove("entry-abi") {
-            if let Json::String(abi_str) = abi_str {
-                match abi_str.parse::<ExternAbi>() {
-                    Ok(abi) => base.options.entry_abi = abi_map.canonize_abi(abi, false).unwrap(),
-                    Err(_) => return Err(format!("{abi_str} is not a valid ExternAbi")),
-                }
-            } else {
-                incorrect_type.push("entry-abi".to_owned())
-            }
+        if let Some(entry_abi) = json.entry_abi {
+            base.options.entry_abi = abi_map.canonize_abi(entry_abi.0, false).unwrap();
         }
 
         base.update_from_cli();
         base.check_consistency(TargetKind::Json)?;
 
-        // Each field should have been read using `Json::remove` so any keys remaining are unused.
-        let remaining_keys = obj.keys();
-        Ok((
-            base,
-            TargetWarnings { unused_fields: remaining_keys.cloned().collect(), incorrect_type },
-        ))
+        Ok((base, TargetWarnings { unused_fields: vec![] }))
     }
 }
 
@@ -877,3 +412,189 @@ impl ToJson for Target {
         Json::Object(d)
     }
 }
+
+#[derive(serde_derive::Deserialize)]
+struct LinkSelfContainedComponentsWrapper {
+    components: Vec<LinkSelfContainedComponents>,
+}
+
+#[derive(serde_derive::Deserialize)]
+#[serde(untagged)]
+enum TargetFamiliesJson {
+    Array(StaticCow<[StaticCow<str>]>),
+    String(StaticCow<str>),
+}
+
+/// `Endian` is in `rustc_abi`, which doesn't have access to the macro and serde.
+struct EndianWrapper(rustc_abi::Endian);
+impl FromStr for EndianWrapper {
+    type Err = String;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        rustc_abi::Endian::from_str(s).map(Self)
+    }
+}
+crate::json::serde_deserialize_from_str!(EndianWrapper);
+
+/// `ExternAbi` is in `rustc_abi`, which doesn't have access to the macro and serde.
+struct ExternAbiWrapper(rustc_abi::ExternAbi);
+impl FromStr for ExternAbiWrapper {
+    type Err = String;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        rustc_abi::ExternAbi::from_str(s)
+            .map(Self)
+            .map_err(|_| format!("{s} is not a valid extern ABI"))
+    }
+}
+crate::json::serde_deserialize_from_str!(ExternAbiWrapper);
+
+#[derive(serde_derive::Deserialize)]
+struct TargetSpecJsonMetadata {
+    description: Option<StaticCow<str>>,
+    tier: Option<u64>,
+    host_tools: Option<bool>,
+    std: Option<bool>,
+}
+
+#[derive(serde_derive::Deserialize)]
+#[serde(rename_all = "kebab-case")]
+// Ensure that all unexpected fields get turned into errors.
+// This helps users stay up to date when the schema changes instead of silently
+// ignoring their old values.
+#[serde(deny_unknown_fields)]
+struct TargetSpecJson {
+    llvm_target: StaticCow<str>,
+    target_pointer_width: String,
+    data_layout: StaticCow<str>,
+    arch: StaticCow<str>,
+
+    metadata: Option<TargetSpecJsonMetadata>,
+
+    // options:
+    target_endian: Option<EndianWrapper>,
+    frame_pointer: Option<FramePointer>,
+    #[serde(rename = "target-c-int-width")]
+    c_int_width: Option<u16>,
+    c_enum_min_bits: Option<u64>,
+    os: Option<StaticCow<str>>,
+    env: Option<StaticCow<str>>,
+    abi: Option<StaticCow<str>>,
+    vendor: Option<StaticCow<str>>,
+    linker: Option<StaticCow<str>>,
+    #[serde(rename = "linker-flavor")]
+    linker_flavor_json: Option<LinkerFlavorCli>,
+    #[serde(rename = "lld-flavor")]
+    lld_flavor_json: Option<LldFlavor>,
+    #[serde(rename = "linker-is-gnu")]
+    linker_is_gnu_json: Option<bool>,
+    #[serde(rename = "pre-link-objects")]
+    pre_link_objects: Option<CrtObjects>,
+    #[serde(rename = "post-link-objects")]
+    post_link_objects: Option<CrtObjects>,
+    #[serde(rename = "pre-link-objects-fallback")]
+    pre_link_objects_self_contained: Option<CrtObjects>,
+    #[serde(rename = "post-link-objects-fallback")]
+    post_link_objects_self_contained: Option<CrtObjects>,
+    #[serde(rename = "crt-objects-fallback")]
+    link_self_contained_backwards_compatible: Option<LinkSelfContainedDefault>,
+    link_self_contained: Option<LinkSelfContainedComponentsWrapper>,
+    #[serde(rename = "pre-link-args")]
+    pre_link_args_json: Option<LinkArgsCli>,
+    #[serde(rename = "late-link-args")]
+    late_link_args_json: Option<LinkArgsCli>,
+    #[serde(rename = "late-link-args-dynamic")]
+    late_link_args_dynamic_json: Option<LinkArgsCli>,
+    #[serde(rename = "late-link-args-static")]
+    late_link_args_static_json: Option<LinkArgsCli>,
+    #[serde(rename = "post-link-args")]
+    post_link_args_json: Option<LinkArgsCli>,
+    link_script: Option<StaticCow<str>>,
+    link_env: Option<Vec<StaticCow<str>>>,
+    link_env_remove: Option<StaticCow<[StaticCow<str>]>>,
+    asm_args: Option<StaticCow<[StaticCow<str>]>>,
+    cpu: Option<StaticCow<str>>,
+    need_explicit_cpu: Option<bool>,
+    features: Option<StaticCow<str>>,
+    dynamic_linking: Option<bool>,
+    direct_access_external_data: Option<bool>,
+    dll_tls_export: Option<bool>,
+    only_cdylib: Option<bool>,
+    executables: Option<bool>,
+    relocation_model: Option<RelocModel>,
+    code_model: Option<CodeModel>,
+    tls_model: Option<TlsModel>,
+    disable_redzone: Option<bool>,
+    function_sections: Option<bool>,
+    dll_prefix: Option<StaticCow<str>>,
+    dll_suffix: Option<StaticCow<str>>,
+    exe_suffix: Option<StaticCow<str>>,
+    staticlib_prefix: Option<StaticCow<str>>,
+    staticlib_suffix: Option<StaticCow<str>>,
+    target_family: Option<TargetFamiliesJson>,
+    abi_return_struct_as_int: Option<bool>,
+    is_like_aix: Option<bool>,
+    is_like_darwin: Option<bool>,
+    is_like_solaris: Option<bool>,
+    is_like_windows: Option<bool>,
+    is_like_msvc: Option<bool>,
+    is_like_wasm: Option<bool>,
+    is_like_android: Option<bool>,
+    binary_format: Option<BinaryFormat>,
+    default_dwarf_version: Option<u32>,
+    allows_weak_linkage: Option<bool>,
+    has_rpath: Option<bool>,
+    no_default_libraries: Option<bool>,
+    position_independent_executables: Option<bool>,
+    static_position_independent_executables: Option<bool>,
+    plt_by_default: Option<bool>,
+    relro_level: Option<RelroLevel>,
+    archive_format: Option<StaticCow<str>>,
+    allow_asm: Option<bool>,
+    main_needs_argc_argv: Option<bool>,
+    has_thread_local: Option<bool>,
+    obj_is_bitcode: Option<bool>,
+    bitcode_llvm_cmdline: Option<StaticCow<str>>,
+    max_atomic_width: Option<u64>,
+    min_atomic_width: Option<u64>,
+    atomic_cas: Option<bool>,
+    panic_strategy: Option<PanicStrategy>,
+    crt_static_allows_dylibs: Option<bool>,
+    crt_static_default: Option<bool>,
+    crt_static_respected: Option<bool>,
+    stack_probes: Option<StackProbeType>,
+    min_global_align: Option<u64>,
+    default_codegen_units: Option<u64>,
+    default_codegen_backend: Option<StaticCow<str>>,
+    trap_unreachable: Option<bool>,
+    requires_lto: Option<bool>,
+    singlethread: Option<bool>,
+    no_builtins: Option<bool>,
+    default_visibility: Option<SymbolVisibility>,
+    emit_debug_gdb_scripts: Option<bool>,
+    requires_uwtable: Option<bool>,
+    default_uwtable: Option<bool>,
+    simd_types_indirect: Option<bool>,
+    limit_rdylib_exports: Option<bool>,
+    override_export_symbols: Option<StaticCow<[StaticCow<str>]>>,
+    merge_functions: Option<MergeFunctions>,
+    #[serde(rename = "target-mcount")]
+    mcount: Option<StaticCow<str>>,
+    llvm_mcount_intrinsic: Option<StaticCow<str>>,
+    llvm_abiname: Option<StaticCow<str>>,
+    llvm_floatabi: Option<FloatAbi>,
+    rustc_abi: Option<RustcAbi>,
+    relax_elf_relocations: Option<bool>,
+    llvm_args: Option<StaticCow<[StaticCow<str>]>>,
+    use_ctors_section: Option<bool>,
+    eh_frame_header: Option<bool>,
+    has_thumb_interworking: Option<bool>,
+    debuginfo_kind: Option<DebuginfoKind>,
+    split_debuginfo: Option<SplitDebuginfo>,
+    supported_split_debuginfo: Option<StaticCow<[SplitDebuginfo]>>,
+    supported_sanitizers: Option<Vec<SanitizerSet>>,
+    generate_arange_section: Option<bool>,
+    supports_stack_protector: Option<bool>,
+    small_data_threshold_support: Option<SmallDataThresholdSupport>,
+    entry_name: Option<StaticCow<str>>,
+    supports_xray: Option<bool>,
+    entry_abi: Option<ExternAbiWrapper>,
+}
diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
index 4bc0d88a910..c64cd9a51b7 100644
--- a/compiler/rustc_target/src/spec/mod.rs
+++ b/compiler/rustc_target/src/spec/mod.rs
@@ -37,6 +37,7 @@
 //!
 //! [JSON]: https://json.org
 
+use core::result::Result;
 use std::borrow::Cow;
 use std::collections::BTreeMap;
 use std::hash::{Hash, Hasher};
@@ -198,18 +199,29 @@ impl LldFlavor {
             LldFlavor::Link => "link",
         }
     }
+}
 
-    fn from_str(s: &str) -> Option<Self> {
-        Some(match s {
+impl FromStr for LldFlavor {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(match s {
             "darwin" => LldFlavor::Ld64,
             "gnu" => LldFlavor::Ld,
             "link" => LldFlavor::Link,
             "wasm" => LldFlavor::Wasm,
-            _ => return None,
+            _ => {
+                return Err(
+                    "invalid value for lld flavor: '{s}', expected one of 'darwin', 'gnu', 'link', 'wasm'"
+                        .into(),
+                );
+            }
         })
     }
 }
 
+crate::json::serde_deserialize_from_str!(LldFlavor);
+
 impl ToJson for LldFlavor {
     fn to_json(&self) -> Json {
         self.as_str().to_json()
@@ -494,19 +506,23 @@ macro_rules! linker_flavor_cli_impls {
                 concat!("one of: ", $($string, " ",)*)
             }
 
-            pub fn from_str(s: &str) -> Option<LinkerFlavorCli> {
-                Some(match s {
-                    $($string => $($flavor)*,)*
-                    _ => return None,
-                })
-            }
-
             pub fn desc(self) -> &'static str {
                 match self {
                     $($($flavor)* => $string,)*
                 }
             }
         }
+
+        impl FromStr for LinkerFlavorCli {
+            type Err = String;
+
+            fn from_str(s: &str) -> Result<LinkerFlavorCli, Self::Err> {
+                Ok(match s {
+                    $($string => $($flavor)*,)*
+                    _ => return Err(format!("invalid linker flavor, allowed values: {}", Self::one_of())),
+                })
+            }
+        }
     )
 }
 
@@ -540,6 +556,8 @@ linker_flavor_cli_impls! {
     (LinkerFlavorCli::Em) "em"
 }
 
+crate::json::serde_deserialize_from_str!(LinkerFlavorCli);
+
 impl ToJson for LinkerFlavorCli {
     fn to_json(&self) -> Json {
         self.desc().to_json()
@@ -573,19 +591,26 @@ pub enum LinkSelfContainedDefault {
 
 /// Parses a backwards-compatible `-Clink-self-contained` option string, without components.
 impl FromStr for LinkSelfContainedDefault {
-    type Err = ();
+    type Err = String;
 
-    fn from_str(s: &str) -> Result<LinkSelfContainedDefault, ()> {
+    fn from_str(s: &str) -> Result<LinkSelfContainedDefault, Self::Err> {
         Ok(match s {
             "false" => LinkSelfContainedDefault::False,
             "true" | "wasm" => LinkSelfContainedDefault::True,
             "musl" => LinkSelfContainedDefault::InferredForMusl,
             "mingw" => LinkSelfContainedDefault::InferredForMingw,
-            _ => return Err(()),
+            _ => {
+                return Err(format!(
+                    "'{s}' is not a valid `-Clink-self-contained` default. \
+                        Use 'false', 'true', 'wasm', 'musl' or 'mingw'",
+                ));
+            }
         })
     }
 }
 
+crate::json::serde_deserialize_from_str!(LinkSelfContainedDefault);
+
 impl ToJson for LinkSelfContainedDefault {
     fn to_json(&self) -> Json {
         match *self {
@@ -652,19 +677,6 @@ bitflags::bitflags! {
 rustc_data_structures::external_bitflags_debug! { LinkSelfContainedComponents }
 
 impl LinkSelfContainedComponents {
-    /// Parses a single `-Clink-self-contained` well-known component, not a set of flags.
-    pub fn from_str(s: &str) -> Option<LinkSelfContainedComponents> {
-        Some(match s {
-            "crto" => LinkSelfContainedComponents::CRT_OBJECTS,
-            "libc" => LinkSelfContainedComponents::LIBC,
-            "unwind" => LinkSelfContainedComponents::UNWIND,
-            "linker" => LinkSelfContainedComponents::LINKER,
-            "sanitizers" => LinkSelfContainedComponents::SANITIZERS,
-            "mingw" => LinkSelfContainedComponents::MINGW,
-            _ => return None,
-        })
-    }
-
     /// Return the component's name.
     ///
     /// Returns `None` if the bitflags aren't a singular component (but a mix of multiple flags).
@@ -708,6 +720,29 @@ impl LinkSelfContainedComponents {
     }
 }
 
+impl FromStr for LinkSelfContainedComponents {
+    type Err = String;
+
+    /// Parses a single `-Clink-self-contained` well-known component, not a set of flags.
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(match s {
+            "crto" => LinkSelfContainedComponents::CRT_OBJECTS,
+            "libc" => LinkSelfContainedComponents::LIBC,
+            "unwind" => LinkSelfContainedComponents::UNWIND,
+            "linker" => LinkSelfContainedComponents::LINKER,
+            "sanitizers" => LinkSelfContainedComponents::SANITIZERS,
+            "mingw" => LinkSelfContainedComponents::MINGW,
+            _ => {
+                return Err(format!(
+                    "'{s}' is not a valid link-self-contained component, expected 'crto', 'libc', 'unwind', 'linker', 'sanitizers', 'mingw'"
+                ));
+            }
+        })
+    }
+}
+
+crate::json::serde_deserialize_from_str!(LinkSelfContainedComponents);
+
 impl ToJson for LinkSelfContainedComponents {
     fn to_json(&self) -> Json {
         let components: Vec<_> = Self::all_components()
@@ -821,6 +856,25 @@ impl PanicStrategy {
     }
 }
 
+impl FromStr for PanicStrategy {
+    type Err = String;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(match s {
+            "unwind" => PanicStrategy::Unwind,
+            "abort" => PanicStrategy::Abort,
+            _ => {
+                return Err(format!(
+                    "'{}' is not a valid value for \
+                    panic-strategy. Use 'unwind' or 'abort'.",
+                    s
+                ));
+            }
+        })
+    }
+}
+
+crate::json::serde_deserialize_from_str!(PanicStrategy);
+
 impl ToJson for PanicStrategy {
     fn to_json(&self) -> Json {
         match *self {
@@ -867,18 +921,24 @@ impl SymbolVisibility {
 }
 
 impl FromStr for SymbolVisibility {
-    type Err = ();
+    type Err = String;
 
-    fn from_str(s: &str) -> Result<SymbolVisibility, ()> {
+    fn from_str(s: &str) -> Result<SymbolVisibility, Self::Err> {
         match s {
             "hidden" => Ok(SymbolVisibility::Hidden),
             "protected" => Ok(SymbolVisibility::Protected),
             "interposable" => Ok(SymbolVisibility::Interposable),
-            _ => Err(()),
+            _ => Err(format!(
+                "'{}' is not a valid value for \
+                    symbol-visibility. Use 'hidden', 'protected, or 'interposable'.",
+                s
+            )),
         }
     }
 }
 
+crate::json::serde_deserialize_from_str!(SymbolVisibility);
+
 impl ToJson for SymbolVisibility {
     fn to_json(&self) -> Json {
         match *self {
@@ -890,19 +950,25 @@ impl ToJson for SymbolVisibility {
 }
 
 impl FromStr for RelroLevel {
-    type Err = ();
+    type Err = String;
 
-    fn from_str(s: &str) -> Result<RelroLevel, ()> {
+    fn from_str(s: &str) -> Result<RelroLevel, Self::Err> {
         match s {
             "full" => Ok(RelroLevel::Full),
             "partial" => Ok(RelroLevel::Partial),
             "off" => Ok(RelroLevel::Off),
             "none" => Ok(RelroLevel::None),
-            _ => Err(()),
+            _ => Err(format!(
+                "'{}' is not a valid value for \
+                        relro-level. Use 'full', 'partial, 'off', or 'none'.",
+                s
+            )),
         }
     }
 }
 
+crate::json::serde_deserialize_from_str!(RelroLevel);
+
 impl ToJson for RelroLevel {
     fn to_json(&self) -> Json {
         match *self {
@@ -923,7 +989,7 @@ pub enum SmallDataThresholdSupport {
 }
 
 impl FromStr for SmallDataThresholdSupport {
-    type Err = ();
+    type Err = String;
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         if s == "none" {
@@ -935,11 +1001,13 @@ impl FromStr for SmallDataThresholdSupport {
         } else if let Some(arg) = s.strip_prefix("llvm-arg=") {
             Ok(Self::LlvmArg(arg.to_string().into()))
         } else {
-            Err(())
+            Err(format!("'{s}' is not a valid value for small-data-threshold-support."))
         }
     }
 }
 
+crate::json::serde_deserialize_from_str!(SmallDataThresholdSupport);
+
 impl ToJson for SmallDataThresholdSupport {
     fn to_json(&self) -> Value {
         match self {
@@ -969,18 +1037,25 @@ impl MergeFunctions {
 }
 
 impl FromStr for MergeFunctions {
-    type Err = ();
+    type Err = String;
 
-    fn from_str(s: &str) -> Result<MergeFunctions, ()> {
+    fn from_str(s: &str) -> Result<MergeFunctions, Self::Err> {
         match s {
             "disabled" => Ok(MergeFunctions::Disabled),
             "trampolines" => Ok(MergeFunctions::Trampolines),
             "aliases" => Ok(MergeFunctions::Aliases),
-            _ => Err(()),
+            _ => Err(format!(
+                "'{}' is not a valid value for \
+                    merge-functions. Use 'disabled', \
+                    'trampolines', or 'aliases'.",
+                s
+            )),
         }
     }
 }
 
+crate::json::serde_deserialize_from_str!(MergeFunctions);
+
 impl ToJson for MergeFunctions {
     fn to_json(&self) -> Json {
         match *self {
@@ -1040,9 +1115,9 @@ impl RelocModel {
 }
 
 impl FromStr for RelocModel {
-    type Err = ();
+    type Err = String;
 
-    fn from_str(s: &str) -> Result<RelocModel, ()> {
+    fn from_str(s: &str) -> Result<RelocModel, Self::Err> {
         Ok(match s {
             "static" => RelocModel::Static,
             "pic" => RelocModel::Pic,
@@ -1051,11 +1126,19 @@ impl FromStr for RelocModel {
             "ropi" => RelocModel::Ropi,
             "rwpi" => RelocModel::Rwpi,
             "ropi-rwpi" => RelocModel::RopiRwpi,
-            _ => return Err(()),
+            _ => {
+                return Err(format!(
+                    "invalid relocation model '{s}'.
+                        Run `rustc --print relocation-models` to \
+                        see the list of supported values.'"
+                ));
+            }
         })
     }
 }
 
+crate::json::serde_deserialize_from_str!(RelocModel);
+
 impl ToJson for RelocModel {
     fn to_json(&self) -> Json {
         self.desc().to_json()
@@ -1072,20 +1155,28 @@ pub enum CodeModel {
 }
 
 impl FromStr for CodeModel {
-    type Err = ();
+    type Err = String;
 
-    fn from_str(s: &str) -> Result<CodeModel, ()> {
+    fn from_str(s: &str) -> Result<CodeModel, Self::Err> {
         Ok(match s {
             "tiny" => CodeModel::Tiny,
             "small" => CodeModel::Small,
             "kernel" => CodeModel::Kernel,
             "medium" => CodeModel::Medium,
             "large" => CodeModel::Large,
-            _ => return Err(()),
+            _ => {
+                return Err(format!(
+                    "'{s}' is not a valid code model. \
+                        Run `rustc --print code-models` to \
+                        see the list of supported values."
+                ));
+            }
         })
     }
 }
 
+crate::json::serde_deserialize_from_str!(CodeModel);
+
 impl ToJson for CodeModel {
     fn to_json(&self) -> Json {
         match *self {
@@ -1107,17 +1198,25 @@ pub enum FloatAbi {
 }
 
 impl FromStr for FloatAbi {
-    type Err = ();
+    type Err = String;
 
-    fn from_str(s: &str) -> Result<FloatAbi, ()> {
+    fn from_str(s: &str) -> Result<FloatAbi, Self::Err> {
         Ok(match s {
             "soft" => FloatAbi::Soft,
             "hard" => FloatAbi::Hard,
-            _ => return Err(()),
+            _ => {
+                return Err(format!(
+                    "'{}' is not a valid value for \
+                        llvm-floatabi. Use 'soft' or 'hard'.",
+                    s
+                ));
+            }
         })
     }
 }
 
+crate::json::serde_deserialize_from_str!(FloatAbi);
+
 impl ToJson for FloatAbi {
     fn to_json(&self) -> Json {
         match *self {
@@ -1138,17 +1237,24 @@ pub enum RustcAbi {
 }
 
 impl FromStr for RustcAbi {
-    type Err = ();
+    type Err = String;
 
-    fn from_str(s: &str) -> Result<RustcAbi, ()> {
+    fn from_str(s: &str) -> Result<RustcAbi, Self::Err> {
         Ok(match s {
             "x86-sse2" => RustcAbi::X86Sse2,
             "x86-softfloat" => RustcAbi::X86Softfloat,
-            _ => return Err(()),
+            _ => {
+                return Err(format!(
+                    "'{s}' is not a valid value for rustc-abi. \
+                        Use 'x86-softfloat' or leave the field unset."
+                ));
+            }
         })
     }
 }
 
+crate::json::serde_deserialize_from_str!(RustcAbi);
+
 impl ToJson for RustcAbi {
     fn to_json(&self) -> Json {
         match *self {
@@ -1169,9 +1275,9 @@ pub enum TlsModel {
 }
 
 impl FromStr for TlsModel {
-    type Err = ();
+    type Err = String;
 
-    fn from_str(s: &str) -> Result<TlsModel, ()> {
+    fn from_str(s: &str) -> Result<TlsModel, Self::Err> {
         Ok(match s {
             // Note the difference "general" vs "global" difference. The model name is "general",
             // but the user-facing option name is "global" for consistency with other compilers.
@@ -1180,11 +1286,19 @@ impl FromStr for TlsModel {
             "initial-exec" => TlsModel::InitialExec,
             "local-exec" => TlsModel::LocalExec,
             "emulated" => TlsModel::Emulated,
-            _ => return Err(()),
+            _ => {
+                return Err(format!(
+                    "'{s}' is not a valid TLS model. \
+                        Run `rustc --print tls-models` to \
+                        see the list of supported values."
+                ));
+            }
         })
     }
 }
 
+crate::json::serde_deserialize_from_str!(TlsModel);
+
 impl ToJson for TlsModel {
     fn to_json(&self) -> Json {
         match *self {
@@ -1230,19 +1344,6 @@ impl LinkOutputKind {
         }
     }
 
-    pub(super) fn from_str(s: &str) -> Option<LinkOutputKind> {
-        Some(match s {
-            "dynamic-nopic-exe" => LinkOutputKind::DynamicNoPicExe,
-            "dynamic-pic-exe" => LinkOutputKind::DynamicPicExe,
-            "static-nopic-exe" => LinkOutputKind::StaticNoPicExe,
-            "static-pic-exe" => LinkOutputKind::StaticPicExe,
-            "dynamic-dylib" => LinkOutputKind::DynamicDylib,
-            "static-dylib" => LinkOutputKind::StaticDylib,
-            "wasi-reactor-exe" => LinkOutputKind::WasiReactorExe,
-            _ => return None,
-        })
-    }
-
     pub fn can_link_dylib(self) -> bool {
         match self {
             LinkOutputKind::StaticNoPicExe | LinkOutputKind::StaticPicExe => false,
@@ -1255,6 +1356,31 @@ impl LinkOutputKind {
     }
 }
 
+impl FromStr for LinkOutputKind {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<LinkOutputKind, Self::Err> {
+        Ok(match s {
+            "dynamic-nopic-exe" => LinkOutputKind::DynamicNoPicExe,
+            "dynamic-pic-exe" => LinkOutputKind::DynamicPicExe,
+            "static-nopic-exe" => LinkOutputKind::StaticNoPicExe,
+            "static-pic-exe" => LinkOutputKind::StaticPicExe,
+            "dynamic-dylib" => LinkOutputKind::DynamicDylib,
+            "static-dylib" => LinkOutputKind::StaticDylib,
+            "wasi-reactor-exe" => LinkOutputKind::WasiReactorExe,
+            _ => {
+                return Err(format!(
+                    "invalid value for CRT object kind. \
+                        Use '(dynamic,static)-(nopic,pic)-exe' or \
+                        '(dynamic,static)-dylib' or 'wasi-reactor-exe'"
+                ));
+            }
+        })
+    }
+}
+
+crate::json::serde_deserialize_from_str!(LinkOutputKind);
+
 impl fmt::Display for LinkOutputKind {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.write_str(self.as_str())
@@ -1290,18 +1416,25 @@ impl DebuginfoKind {
 }
 
 impl FromStr for DebuginfoKind {
-    type Err = ();
+    type Err = String;
 
-    fn from_str(s: &str) -> Result<Self, ()> {
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
         Ok(match s {
             "dwarf" => DebuginfoKind::Dwarf,
             "dwarf-dsym" => DebuginfoKind::DwarfDsym,
             "pdb" => DebuginfoKind::Pdb,
-            _ => return Err(()),
+            _ => {
+                return Err(format!(
+                    "'{s}' is not a valid value for debuginfo-kind. Use 'dwarf', \
+                        'dwarf-dsym' or 'pdb'."
+                ));
+            }
         })
     }
 }
 
+crate::json::serde_deserialize_from_str!(DebuginfoKind);
+
 impl ToJson for DebuginfoKind {
     fn to_json(&self) -> Json {
         self.as_str().to_json()
@@ -1354,18 +1487,25 @@ impl SplitDebuginfo {
 }
 
 impl FromStr for SplitDebuginfo {
-    type Err = ();
+    type Err = String;
 
-    fn from_str(s: &str) -> Result<Self, ()> {
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
         Ok(match s {
             "off" => SplitDebuginfo::Off,
             "unpacked" => SplitDebuginfo::Unpacked,
             "packed" => SplitDebuginfo::Packed,
-            _ => return Err(()),
+            _ => {
+                return Err(format!(
+                    "'{s}' is not a valid value for \
+                        split-debuginfo. Use 'off', 'unpacked', or 'packed'.",
+                ));
+            }
         })
     }
 }
 
+crate::json::serde_deserialize_from_str!(SplitDebuginfo);
+
 impl ToJson for SplitDebuginfo {
     fn to_json(&self) -> Json {
         self.as_str().to_json()
@@ -1378,7 +1518,9 @@ impl fmt::Display for SplitDebuginfo {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Debug, PartialEq, Eq, serde_derive::Deserialize)]
+#[serde(tag = "kind")]
+#[serde(rename_all = "kebab-case")]
 pub enum StackProbeType {
     /// Don't emit any stack probes.
     None,
@@ -1390,44 +1532,10 @@ pub enum StackProbeType {
     Call,
     /// Use inline option for LLVM versions later than specified in `min_llvm_version_for_inline`
     /// and call `__rust_probestack` otherwise.
-    InlineOrCall { min_llvm_version_for_inline: (u32, u32, u32) },
-}
-
-impl StackProbeType {
-    fn from_json(json: &Json) -> Result<Self, String> {
-        let object = json.as_object().ok_or_else(|| "expected a JSON object")?;
-        let kind = object
-            .get("kind")
-            .and_then(|o| o.as_str())
-            .ok_or_else(|| "expected `kind` to be a string")?;
-        match kind {
-            "none" => Ok(StackProbeType::None),
-            "inline" => Ok(StackProbeType::Inline),
-            "call" => Ok(StackProbeType::Call),
-            "inline-or-call" => {
-                let min_version = object
-                    .get("min-llvm-version-for-inline")
-                    .and_then(|o| o.as_array())
-                    .ok_or_else(|| "expected `min-llvm-version-for-inline` to be an array")?;
-                let mut iter = min_version.into_iter().map(|v| {
-                    let int = v.as_u64().ok_or_else(
-                        || "expected `min-llvm-version-for-inline` values to be integers",
-                    )?;
-                    u32::try_from(int)
-                        .map_err(|_| "`min-llvm-version-for-inline` values don't convert to u32")
-                });
-                let min_llvm_version_for_inline = (
-                    iter.next().unwrap_or(Ok(11))?,
-                    iter.next().unwrap_or(Ok(0))?,
-                    iter.next().unwrap_or(Ok(0))?,
-                );
-                Ok(StackProbeType::InlineOrCall { min_llvm_version_for_inline })
-            }
-            _ => Err(String::from(
-                "`kind` expected to be one of `none`, `inline`, `call` or `inline-or-call`",
-            )),
-        }
-    }
+    InlineOrCall {
+        #[serde(rename = "min-llvm-version-for-inline")]
+        min_llvm_version_for_inline: (u32, u32, u32),
+    },
 }
 
 impl ToJson for StackProbeType {
@@ -1549,6 +1657,29 @@ impl fmt::Display for SanitizerSet {
     }
 }
 
+impl FromStr for SanitizerSet {
+    type Err = String;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(match s {
+            "address" => SanitizerSet::ADDRESS,
+            "cfi" => SanitizerSet::CFI,
+            "dataflow" => SanitizerSet::DATAFLOW,
+            "kcfi" => SanitizerSet::KCFI,
+            "kernel-address" => SanitizerSet::KERNELADDRESS,
+            "leak" => SanitizerSet::LEAK,
+            "memory" => SanitizerSet::MEMORY,
+            "memtag" => SanitizerSet::MEMTAG,
+            "safestack" => SanitizerSet::SAFESTACK,
+            "shadow-call-stack" => SanitizerSet::SHADOWCALLSTACK,
+            "thread" => SanitizerSet::THREAD,
+            "hwaddress" => SanitizerSet::HWADDRESS,
+            s => return Err(format!("unknown sanitizer {s}")),
+        })
+    }
+}
+
+crate::json::serde_deserialize_from_str!(SanitizerSet);
+
 impl ToJson for SanitizerSet {
     fn to_json(&self) -> Json {
         self.into_iter()
@@ -1587,17 +1718,19 @@ impl FramePointer {
 }
 
 impl FromStr for FramePointer {
-    type Err = ();
-    fn from_str(s: &str) -> Result<Self, ()> {
+    type Err = String;
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
         Ok(match s {
             "always" => Self::Always,
             "non-leaf" => Self::NonLeaf,
             "may-omit" => Self::MayOmit,
-            _ => return Err(()),
+            _ => return Err(format!("'{s}' is not a valid value for frame-pointer")),
         })
     }
 }
 
+crate::json::serde_deserialize_from_str!(FramePointer);
+
 impl ToJson for FramePointer {
     fn to_json(&self) -> Json {
         match *self {
@@ -1685,7 +1818,7 @@ impl BinaryFormat {
 }
 
 impl FromStr for BinaryFormat {
-    type Err = ();
+    type Err = String;
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         match s {
             "coff" => Ok(Self::Coff),
@@ -1693,11 +1826,16 @@ impl FromStr for BinaryFormat {
             "mach-o" => Ok(Self::MachO),
             "wasm" => Ok(Self::Wasm),
             "xcoff" => Ok(Self::Xcoff),
-            _ => Err(()),
+            _ => Err(format!(
+                "'{s}' is not a valid value for binary_format. \
+                    Use 'coff', 'elf', 'mach-o', 'wasm' or 'xcoff' "
+            )),
         }
     }
 }
 
+crate::json::serde_deserialize_from_str!(BinaryFormat);
+
 impl ToJson for BinaryFormat {
     fn to_json(&self) -> Json {
         match self {
@@ -2130,12 +2268,11 @@ pub(crate) use cvs;
 #[derive(Debug, PartialEq)]
 pub struct TargetWarnings {
     unused_fields: Vec<String>,
-    incorrect_type: Vec<String>,
 }
 
 impl TargetWarnings {
     pub fn empty() -> Self {
-        Self { unused_fields: Vec::new(), incorrect_type: Vec::new() }
+        Self { unused_fields: Vec::new() }
     }
 
     pub fn warning_messages(&self) -> Vec<String> {
@@ -2146,12 +2283,6 @@ impl TargetWarnings {
                 self.unused_fields.join(", ")
             ));
         }
-        if !self.incorrect_type.is_empty() {
-            warnings.push(format!(
-                "target json file contains fields whose value doesn't have the correct json type: {}",
-                self.incorrect_type.join(", ")
-            ));
-        }
         warnings
     }
 }
@@ -3325,7 +3456,8 @@ impl Target {
     /// Test target self-consistency and JSON encoding/decoding roundtrip.
     #[cfg(test)]
     fn test_target(mut self) {
-        let recycled_target = Target::from_json(self.to_json()).map(|(j, _)| j);
+        let recycled_target =
+            Target::from_json(&serde_json::to_string(&self.to_json()).unwrap()).map(|(j, _)| j);
         self.update_to_cli();
         self.check_consistency(TargetKind::Builtin).unwrap();
         assert_eq!(recycled_target, Ok(self));
@@ -3373,8 +3505,7 @@ impl Target {
 
         fn load_file(path: &Path) -> Result<(Target, TargetWarnings), String> {
             let contents = fs::read_to_string(path).map_err(|e| e.to_string())?;
-            let obj = serde_json::from_str(&contents).map_err(|e| e.to_string())?;
-            Target::from_json(obj)
+            Target::from_json(&contents)
         }
 
         match *target_tuple {
@@ -3422,10 +3553,7 @@ impl Target {
                     Err(format!("could not find specification for target {target_tuple:?}"))
                 }
             }
-            TargetTuple::TargetJson { ref contents, .. } => {
-                let obj = serde_json::from_str(contents).map_err(|e| e.to_string())?;
-                Target::from_json(obj)
-            }
+            TargetTuple::TargetJson { ref contents, .. } => Target::from_json(contents),
         }
     }
 
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_musl.rs
index 58daaa03675..478726fbef6 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_musl.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_musl.rs
@@ -6,7 +6,7 @@ pub(crate) fn target() -> Target {
     let mut base = base::linux_musl::opts();
     base.max_atomic_width = Some(128);
     base.supports_xray = true;
-    base.features = "+v8a".into();
+    base.features = "+v8a,+outline-atomics".into();
     base.stack_probes = StackProbeType::Inline;
     base.supported_sanitizers = SanitizerSet::ADDRESS
         | SanitizerSet::CFI
diff --git a/compiler/rustc_target/src/tests.rs b/compiler/rustc_target/src/tests.rs
index 76375170db6..ee847a84007 100644
--- a/compiler/rustc_target/src/tests.rs
+++ b/compiler/rustc_target/src/tests.rs
@@ -2,8 +2,7 @@ use crate::spec::Target;
 
 #[test]
 fn report_unused_fields() {
-    let json = serde_json::from_str(
-        r#"
+    let json = r#"
     {
         "arch": "powerpc64",
         "data-layout": "e-m:e-i64:64-n32:64",
@@ -11,47 +10,8 @@ fn report_unused_fields() {
         "target-pointer-width": "64",
         "code-mode": "foo"
     }
-    "#,
-    )
-    .unwrap();
-    let warnings = Target::from_json(json).unwrap().1;
-    assert_eq!(warnings.warning_messages().len(), 1);
-    assert!(warnings.warning_messages().join("\n").contains("code-mode"));
-}
-
-#[test]
-fn report_incorrect_json_type() {
-    let json = serde_json::from_str(
-        r#"
-    {
-        "arch": "powerpc64",
-        "data-layout": "e-m:e-i64:64-n32:64",
-        "llvm-target": "powerpc64le-elf",
-        "target-pointer-width": "64",
-        "link-env-remove": "foo"
-    }
-    "#,
-    )
-    .unwrap();
-    let warnings = Target::from_json(json).unwrap().1;
-    assert_eq!(warnings.warning_messages().len(), 1);
-    assert!(warnings.warning_messages().join("\n").contains("link-env-remove"));
-}
-
-#[test]
-fn no_warnings_for_valid_target() {
-    let json = serde_json::from_str(
-        r#"
-    {
-        "arch": "powerpc64",
-        "data-layout": "e-m:e-i64:64-n32:64",
-        "llvm-target": "powerpc64le-elf",
-        "target-pointer-width": "64",
-        "link-env-remove": ["foo"]
-    }
-    "#,
-    )
-    .unwrap();
-    let warnings = Target::from_json(json).unwrap().1;
-    assert_eq!(warnings.warning_messages().len(), 0);
+    "#;
+    let result = Target::from_json(json);
+    eprintln!("{result:#?}");
+    assert!(result.is_err());
 }
diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs
index 163e2b30883..79f7e228e2a 100644
--- a/compiler/rustc_ty_utils/src/layout.rs
+++ b/compiler/rustc_ty_utils/src/layout.rs
@@ -603,12 +603,6 @@ fn layout_of_uncached<'tcx>(
                     .flatten()
             };
 
-            let dont_niche_optimize_enum = def.repr().inhibit_enum_layout_opt()
-                || def
-                    .variants()
-                    .iter_enumerated()
-                    .any(|(i, v)| v.discr != ty::VariantDiscr::Relative(i.as_u32()));
-
             let maybe_unsized = def.is_struct()
                 && def.non_enum_variant().tail_opt().is_some_and(|last_field| {
                     let typing_env = ty::TypingEnv::post_analysis(tcx, def.did());
@@ -625,7 +619,6 @@ fn layout_of_uncached<'tcx>(
                     tcx.layout_scalar_valid_range(def.did()),
                     get_discriminant_type,
                     discriminants_iter(),
-                    dont_niche_optimize_enum,
                     !maybe_unsized,
                 )
                 .map_err(|err| map_error(cx, ty, err))?;
@@ -651,7 +644,6 @@ fn layout_of_uncached<'tcx>(
                     tcx.layout_scalar_valid_range(def.did()),
                     get_discriminant_type,
                     discriminants_iter(),
-                    dont_niche_optimize_enum,
                     !maybe_unsized,
                 ) else {
                     bug!("failed to compute unsized layout of {ty:?}");
diff --git a/compiler/rustc_ty_utils/src/ty.rs b/compiler/rustc_ty_utils/src/ty.rs
index f8d793464a9..18f10da8505 100644
--- a/compiler/rustc_ty_utils/src/ty.rs
+++ b/compiler/rustc_ty_utils/src/ty.rs
@@ -80,8 +80,11 @@ fn sizedness_constraint_for_ty<'tcx>(
 
 fn defaultness(tcx: TyCtxt<'_>, def_id: LocalDefId) -> hir::Defaultness {
     match tcx.hir_node_by_def_id(def_id) {
-        hir::Node::Item(hir::Item { kind: hir::ItemKind::Impl(impl_), .. }) => impl_.defaultness,
-        hir::Node::ImplItem(hir::ImplItem { defaultness, .. })
+        hir::Node::Item(hir::Item {
+            kind: hir::ItemKind::Impl(hir::Impl { defaultness, of_trait: Some(_), .. }),
+            ..
+        })
+        | hir::Node::ImplItem(hir::ImplItem { defaultness, .. })
         | hir::Node::TraitItem(hir::TraitItem { defaultness, .. }) => *defaultness,
         node => {
             bug!("`defaultness` called on {:?}", node);
diff --git a/compiler/rustc_type_ir/src/relate.rs b/compiler/rustc_type_ir/src/relate.rs
index e3c4a793b37..3a00fe89360 100644
--- a/compiler/rustc_type_ir/src/relate.rs
+++ b/compiler/rustc_type_ir/src/relate.rs
@@ -400,8 +400,12 @@ pub fn structurally_relate_tys<I: Interner, R: TypeRelation<I>>(
         (ty::Placeholder(p1), ty::Placeholder(p2)) if p1 == p2 => Ok(a),
 
         (ty::Adt(a_def, a_args), ty::Adt(b_def, b_args)) if a_def == b_def => {
-            let args = relation.relate_item_args(a_def.def_id(), a_args, b_args)?;
-            Ok(Ty::new_adt(cx, a_def, args))
+            Ok(if a_args.is_empty() {
+                a
+            } else {
+                let args = relation.relate_item_args(a_def.def_id(), a_args, b_args)?;
+                if args == a_args { a } else { Ty::new_adt(cx, a_def, args) }
+            })
         }
 
         (ty::Foreign(a_id), ty::Foreign(b_id)) if a_id == b_id => Ok(Ty::new_foreign(cx, a_id)),
@@ -515,8 +519,12 @@ pub fn structurally_relate_tys<I: Interner, R: TypeRelation<I>>(
         }
 
         (ty::FnDef(a_def_id, a_args), ty::FnDef(b_def_id, b_args)) if a_def_id == b_def_id => {
-            let args = relation.relate_item_args(a_def_id, a_args, b_args)?;
-            Ok(Ty::new_fn_def(cx, a_def_id, args))
+            Ok(if a_args.is_empty() {
+                a
+            } else {
+                let args = relation.relate_item_args(a_def_id, a_args, b_args)?;
+                if args == a_args { a } else { Ty::new_fn_def(cx, a_def_id, args) }
+            })
         }
 
         (ty::FnPtr(a_sig_tys, a_hdr), ty::FnPtr(b_sig_tys, b_hdr)) => {