about summary refs log tree commit diff
path: root/compiler
diff options
context:
space:
mode:
Diffstat (limited to 'compiler')
-rw-r--r--compiler/rustc_ast/src/ast.rs4
-rw-r--r--compiler/rustc_ast/src/token.rs1
-rw-r--r--compiler/rustc_ast_lowering/Cargo.toml1
-rw-r--r--compiler/rustc_ast_lowering/src/expr.rs3
-rw-r--r--compiler/rustc_ast_passes/Cargo.toml1
-rw-r--r--compiler/rustc_ast_passes/messages.ftl23
-rw-r--r--compiler/rustc_ast_passes/src/ast_validation.rs87
-rw-r--r--compiler/rustc_ast_passes/src/errors.rs22
-rw-r--r--compiler/rustc_ast_pretty/src/pprust/state/expr.rs4
-rw-r--r--compiler/rustc_attr_data_structures/src/attributes.rs9
-rw-r--r--compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs24
-rw-r--r--compiler/rustc_attr_parsing/src/attributes/loop_match.rs31
-rw-r--r--compiler/rustc_attr_parsing/src/attributes/mod.rs1
-rw-r--r--compiler/rustc_attr_parsing/src/context.rs8
-rw-r--r--compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs11
-rw-r--r--compiler/rustc_borrowck/src/diagnostics/mod.rs43
-rw-r--r--compiler/rustc_borrowck/src/diagnostics/move_errors.rs10
-rw-r--r--compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs6
-rw-r--r--compiler/rustc_borrowck/src/diagnostics/region_errors.rs4
-rw-r--r--compiler/rustc_borrowck/src/diagnostics/region_name.rs4
-rw-r--r--compiler/rustc_borrowck/src/lib.rs30
-rw-r--r--compiler/rustc_builtin_macros/messages.ftl4
-rw-r--r--compiler/rustc_builtin_macros/src/concat_idents.rs71
-rw-r--r--compiler/rustc_builtin_macros/src/errors.rs21
-rw-r--r--compiler/rustc_builtin_macros/src/lib.rs2
-rw-r--r--compiler/rustc_codegen_cranelift/Cargo.lock84
-rw-r--r--compiler/rustc_codegen_cranelift/Cargo.toml24
-rw-r--r--compiler/rustc_codegen_cranelift/build_system/abi_cafe.rs8
-rw-r--r--compiler/rustc_codegen_cranelift/patches/0002-abi-cafe-Disable-broken-tests.patch69
-rw-r--r--compiler/rustc_codegen_cranelift/rust-toolchain2
-rw-r--r--compiler/rustc_codegen_cranelift/scripts/abi-cafe-rules.toml17
-rwxr-xr-xcompiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh14
-rw-r--r--compiler/rustc_codegen_cranelift/src/constant.rs2
-rw-r--r--compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs7
-rw-r--r--compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs7
-rw-r--r--compiler/rustc_codegen_ssa/src/back/link.rs12
-rw-r--r--compiler/rustc_codegen_ssa/src/back/linker.rs131
-rw-r--r--compiler/rustc_codegen_ssa/src/back/symbol_export.rs12
-rw-r--r--compiler/rustc_codegen_ssa/src/base.rs47
-rw-r--r--compiler/rustc_codegen_ssa/src/codegen_attrs.rs54
-rw-r--r--compiler/rustc_codegen_ssa/src/lib.rs2
-rw-r--r--compiler/rustc_codegen_ssa/src/mir/rvalue.rs10
-rw-r--r--compiler/rustc_const_eval/src/check_consts/ops.rs12
-rw-r--r--compiler/rustc_driver_impl/src/lib.rs10
-rw-r--r--compiler/rustc_error_messages/Cargo.toml1
-rw-r--r--compiler/rustc_error_messages/src/lib.rs8
-rw-r--r--compiler/rustc_expand/messages.ftl2
-rw-r--r--compiler/rustc_expand/src/config.rs1
-rw-r--r--compiler/rustc_expand/src/errors.rs3
-rw-r--r--compiler/rustc_expand/src/mbe.rs10
-rw-r--r--compiler/rustc_expand/src/mbe/diagnostics.rs1
-rw-r--r--compiler/rustc_expand/src/mbe/macro_check.rs12
-rw-r--r--compiler/rustc_expand/src/mbe/macro_parser.rs53
-rw-r--r--compiler/rustc_expand/src/mbe/macro_rules.rs49
-rw-r--r--compiler/rustc_expand/src/mbe/quoted.rs128
-rw-r--r--compiler/rustc_expand/src/mbe/transcribe.rs4
-rw-r--r--compiler/rustc_expand/src/stats.rs28
-rw-r--r--compiler/rustc_feature/src/builtin_attrs.rs13
-rw-r--r--compiler/rustc_feature/src/removed.rs14
-rw-r--r--compiler/rustc_feature/src/unstable.rs2
-rw-r--r--compiler/rustc_hir_analysis/messages.ftl2
-rw-r--r--compiler/rustc_hir_analysis/src/check/entry.rs9
-rw-r--r--compiler/rustc_hir_pretty/src/lib.rs4
-rw-r--r--compiler/rustc_hir_typeck/messages.ftl3
-rw-r--r--compiler/rustc_hir_typeck/src/errors.rs7
-rw-r--r--compiler/rustc_hir_typeck/src/expr.rs8
-rw-r--r--compiler/rustc_hir_typeck/src/loops.rs70
-rw-r--r--compiler/rustc_interface/src/interface.rs13
-rw-r--r--compiler/rustc_interface/src/passes.rs6
-rw-r--r--compiler/rustc_interface/src/tests.rs9
-rw-r--r--compiler/rustc_interface/src/util.rs36
-rw-r--r--compiler/rustc_lint/src/builtin.rs4
-rw-r--r--compiler/rustc_lint/src/context.rs14
-rw-r--r--compiler/rustc_lint/src/lints.rs25
-rw-r--r--compiler/rustc_lint/src/nonstandard_style.rs107
-rw-r--r--compiler/rustc_metadata/src/locator.rs2
-rw-r--r--compiler/rustc_metadata/src/rmeta/encoder.rs37
-rw-r--r--compiler/rustc_middle/src/middle/exported_symbols.rs2
-rw-r--r--compiler/rustc_middle/src/thir.rs13
-rw-r--r--compiler/rustc_middle/src/thir/visit.rs3
-rw-r--r--compiler/rustc_mir_build/Cargo.toml1
-rw-r--r--compiler/rustc_mir_build/messages.ftl33
-rw-r--r--compiler/rustc_mir_build/src/builder/expr/as_place.rs2
-rw-r--r--compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs2
-rw-r--r--compiler/rustc_mir_build/src/builder/expr/category.rs2
-rw-r--r--compiler/rustc_mir_build/src/builder/expr/into.rs122
-rw-r--r--compiler/rustc_mir_build/src/builder/expr/stmt.rs3
-rw-r--r--compiler/rustc_mir_build/src/builder/matches/mod.rs144
-rw-r--r--compiler/rustc_mir_build/src/builder/scope.rs264
-rw-r--r--compiler/rustc_mir_build/src/check_unsafety.rs2
-rw-r--r--compiler/rustc_mir_build/src/errors.rs77
-rw-r--r--compiler/rustc_mir_build/src/thir/cx/expr.rs192
-rw-r--r--compiler/rustc_mir_build/src/thir/pattern/check_match.rs7
-rw-r--r--compiler/rustc_mir_build/src/thir/print.rs21
-rw-r--r--compiler/rustc_parse/src/parser/expr.rs7
-rw-r--r--compiler/rustc_parse/src/validate_attr.rs1
-rw-r--r--compiler/rustc_passes/messages.ftl9
-rw-r--r--compiler/rustc_passes/src/check_attr.rs42
-rw-r--r--compiler/rustc_passes/src/errors.rs18
-rw-r--r--compiler/rustc_passes/src/input_stats.rs61
-rw-r--r--compiler/rustc_pattern_analysis/src/constructor.rs3
-rw-r--r--compiler/rustc_resolve/src/imports.rs18
-rw-r--r--compiler/rustc_session/Cargo.toml1
-rw-r--r--compiler/rustc_session/src/config.rs31
-rw-r--r--compiler/rustc_session/src/filesearch.rs18
-rw-r--r--compiler/rustc_session/src/options.rs2
-rw-r--r--compiler/rustc_session/src/session.rs16
-rw-r--r--compiler/rustc_span/src/symbol.rs2
-rw-r--r--compiler/rustc_ty_utils/src/consts.rs8
109 files changed, 2008 insertions, 753 deletions
diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
index 11afd359e5a..ab8dac16026 100644
--- a/compiler/rustc_ast/src/ast.rs
+++ b/compiler/rustc_ast/src/ast.rs
@@ -904,6 +904,10 @@ pub enum BorrowKind {
     /// The resulting type is either `*const T` or `*mut T`
     /// where `T = typeof($expr)`.
     Raw,
+    /// A pinned borrow, `&pin const $expr` or `&pin mut $expr`.
+    /// The resulting type is either `Pin<&'a T>` or `Pin<&'a mut T>`
+    /// where `T = typeof($expr)` and `'a` is some lifetime.
+    Pin,
 }
 
 #[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs
index 54781e8235e..9b4535dcfbc 100644
--- a/compiler/rustc_ast/src/token.rs
+++ b/compiler/rustc_ast/src/token.rs
@@ -1085,6 +1085,7 @@ pub enum NtExprKind {
     Expr2021 { inferred: bool },
 }
 
+/// A macro nonterminal, known in documentation as a fragment specifier.
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, HashStable_Generic)]
 pub enum NonterminalKind {
     Item,
diff --git a/compiler/rustc_ast_lowering/Cargo.toml b/compiler/rustc_ast_lowering/Cargo.toml
index 6ac258155fe..dc571f5c367 100644
--- a/compiler/rustc_ast_lowering/Cargo.toml
+++ b/compiler/rustc_ast_lowering/Cargo.toml
@@ -11,6 +11,7 @@ doctest = false
 rustc_abi = { path = "../rustc_abi" }
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
+rustc_attr_data_structures = { path = "../rustc_attr_data_structures" }
 rustc_attr_parsing = { path = "../rustc_attr_parsing" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index f297bf9f4cf..c2140514e31 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -4,6 +4,7 @@ use std::sync::Arc;
 use rustc_ast::ptr::P as AstP;
 use rustc_ast::*;
 use rustc_ast_pretty::pprust::expr_to_string;
+use rustc_attr_data_structures::{AttributeKind, find_attr};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_hir as hir;
 use rustc_hir::HirId;
@@ -831,7 +832,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
     ) {
         if self.tcx.features().async_fn_track_caller()
             && let Some(attrs) = self.attrs.get(&outer_hir_id.local_id)
-            && attrs.into_iter().any(|attr| attr.has_name(sym::track_caller))
+            && find_attr!(*attrs, AttributeKind::TrackCaller(_))
         {
             let unstable_span = self.mark_span_with_reason(
                 DesugaringKind::Async,
diff --git a/compiler/rustc_ast_passes/Cargo.toml b/compiler/rustc_ast_passes/Cargo.toml
index c738cb2aa2f..1940628b44a 100644
--- a/compiler/rustc_ast_passes/Cargo.toml
+++ b/compiler/rustc_ast_passes/Cargo.toml
@@ -18,5 +18,6 @@ rustc_macros = { path = "../rustc_macros" }
 rustc_parse = { path = "../rustc_parse" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
+rustc_target = { path = "../rustc_target" }
 thin-vec = "0.2.12"
 # tidy-alphabetical-end
diff --git a/compiler/rustc_ast_passes/messages.ftl b/compiler/rustc_ast_passes/messages.ftl
index 9a267501230..6eddce7b590 100644
--- a/compiler/rustc_ast_passes/messages.ftl
+++ b/compiler/rustc_ast_passes/messages.ftl
@@ -1,20 +1,25 @@
-ast_passes_abi_custom_coroutine =
-    functions with the `"custom"` ABI cannot be `{$coroutine_kind_str}`
+ast_passes_abi_cannot_be_coroutine =
+    functions with the {$abi} ABI cannot be `{$coroutine_kind_str}`
     .suggestion = remove the `{$coroutine_kind_str}` keyword from this definiton
 
-ast_passes_abi_custom_invalid_signature =
-    invalid signature for `extern "custom"` function
-    .note = functions with the `"custom"` ABI cannot have any parameters or return type
-    .suggestion = remove the parameters and return type
-
 ast_passes_abi_custom_safe_foreign_function =
-    foreign functions with the `"custom"` ABI cannot be safe
+    foreign functions with the "custom" ABI cannot be safe
     .suggestion = remove the `safe` keyword from this definition
 
 ast_passes_abi_custom_safe_function =
-    functions with the `"custom"` ABI must be unsafe
+    functions with the "custom" ABI must be unsafe
     .suggestion = add the `unsafe` keyword to this definition
 
+ast_passes_abi_must_not_have_parameters_or_return_type=
+    invalid signature for `extern {$abi}` function
+    .note = functions with the {$abi} ABI cannot have any parameters or return type
+    .suggestion = remove the parameters and return type
+
+ast_passes_abi_must_not_have_return_type=
+    invalid signature for `extern {$abi}` function
+    .note = functions with the "custom" ABI cannot have a return type
+    .help = remove the return type
+
 ast_passes_assoc_const_without_body =
     associated constant in `impl` without body
     .suggestion = provide a definition for the constant
diff --git a/compiler/rustc_ast_passes/src/ast_validation.rs b/compiler/rustc_ast_passes/src/ast_validation.rs
index b69a91e2f5f..da248251203 100644
--- a/compiler/rustc_ast_passes/src/ast_validation.rs
+++ b/compiler/rustc_ast_passes/src/ast_validation.rs
@@ -21,7 +21,7 @@ use std::ops::{Deref, DerefMut};
 use std::str::FromStr;
 
 use itertools::{Either, Itertools};
-use rustc_abi::ExternAbi;
+use rustc_abi::{CanonAbi, ExternAbi, InterruptKind};
 use rustc_ast::ptr::P;
 use rustc_ast::visit::{AssocCtxt, BoundKind, FnCtxt, FnKind, Visitor, walk_list};
 use rustc_ast::*;
@@ -37,6 +37,7 @@ use rustc_session::lint::builtin::{
 };
 use rustc_session::lint::{BuiltinLintDiag, LintBuffer};
 use rustc_span::{Ident, Span, kw, sym};
+use rustc_target::spec::{AbiMap, AbiMapping};
 use thin_vec::thin_vec;
 
 use crate::errors::{self, TildeConstReason};
@@ -365,31 +366,77 @@ impl<'a> AstValidator<'a> {
         }
     }
 
-    /// An `extern "custom"` function must be unsafe, and must not have any parameters or return
-    /// type.
-    fn check_custom_abi(&self, ctxt: FnCtxt, ident: &Ident, sig: &FnSig) {
+    /// Check that the signature of this function does not violate the constraints of its ABI.
+    fn check_extern_fn_signature(&self, abi: ExternAbi, ctxt: FnCtxt, ident: &Ident, sig: &FnSig) {
+        match AbiMap::from_target(&self.sess.target).canonize_abi(abi, false) {
+            AbiMapping::Direct(canon_abi) | AbiMapping::Deprecated(canon_abi) => {
+                match canon_abi {
+                    CanonAbi::C
+                    | CanonAbi::Rust
+                    | CanonAbi::RustCold
+                    | CanonAbi::Arm(_)
+                    | CanonAbi::GpuKernel
+                    | CanonAbi::X86(_) => { /* nothing to check */ }
+
+                    CanonAbi::Custom => {
+                        // An `extern "custom"` function must be unsafe.
+                        self.reject_safe_fn(abi, ctxt, sig);
+
+                        // An `extern "custom"` function cannot be `async` and/or `gen`.
+                        self.reject_coroutine(abi, sig);
+
+                        // An `extern "custom"` function must have type `fn()`.
+                        self.reject_params_or_return(abi, ident, sig);
+                    }
+
+                    CanonAbi::Interrupt(interrupt_kind) => {
+                        // An interrupt handler cannot be `async` and/or `gen`.
+                        self.reject_coroutine(abi, sig);
+
+                        if let InterruptKind::X86 = interrupt_kind {
+                            // "x86-interrupt" is special because it does have arguments.
+                            // FIXME(workingjubilee): properly lint on acceptable input types.
+                            if let FnRetTy::Ty(ref ret_ty) = sig.decl.output {
+                                self.dcx().emit_err(errors::AbiMustNotHaveReturnType {
+                                    span: ret_ty.span,
+                                    abi,
+                                });
+                            }
+                        } else {
+                            // An `extern "interrupt"` function must have type `fn()`.
+                            self.reject_params_or_return(abi, ident, sig);
+                        }
+                    }
+                }
+            }
+            AbiMapping::Invalid => { /* ignore */ }
+        }
+    }
+
+    fn reject_safe_fn(&self, abi: ExternAbi, ctxt: FnCtxt, sig: &FnSig) {
         let dcx = self.dcx();
 
-        // An `extern "custom"` function must be unsafe.
         match sig.header.safety {
             Safety::Unsafe(_) => { /* all good */ }
             Safety::Safe(safe_span) => {
-                let safe_span =
-                    self.sess.psess.source_map().span_until_non_whitespace(safe_span.to(sig.span));
+                let source_map = self.sess.psess.source_map();
+                let safe_span = source_map.span_until_non_whitespace(safe_span.to(sig.span));
                 dcx.emit_err(errors::AbiCustomSafeForeignFunction { span: sig.span, safe_span });
             }
             Safety::Default => match ctxt {
                 FnCtxt::Foreign => { /* all good */ }
                 FnCtxt::Free | FnCtxt::Assoc(_) => {
-                    self.dcx().emit_err(errors::AbiCustomSafeFunction {
+                    dcx.emit_err(errors::AbiCustomSafeFunction {
                         span: sig.span,
+                        abi,
                         unsafe_span: sig.span.shrink_to_lo(),
                     });
                 }
             },
         }
+    }
 
-        // An `extern "custom"` function cannot be `async` and/or `gen`.
+    fn reject_coroutine(&self, abi: ExternAbi, sig: &FnSig) {
         if let Some(coroutine_kind) = sig.header.coroutine_kind {
             let coroutine_kind_span = self
                 .sess
@@ -397,14 +444,16 @@ impl<'a> AstValidator<'a> {
                 .source_map()
                 .span_until_non_whitespace(coroutine_kind.span().to(sig.span));
 
-            self.dcx().emit_err(errors::AbiCustomCoroutine {
+            self.dcx().emit_err(errors::AbiCannotBeCoroutine {
                 span: sig.span,
+                abi,
                 coroutine_kind_span,
                 coroutine_kind_str: coroutine_kind.as_str(),
             });
         }
+    }
 
-        // An `extern "custom"` function must not have any parameters or return type.
+    fn reject_params_or_return(&self, abi: ExternAbi, ident: &Ident, sig: &FnSig) {
         let mut spans: Vec<_> = sig.decl.inputs.iter().map(|p| p.span).collect();
         if let FnRetTy::Ty(ref ret_ty) = sig.decl.output {
             spans.push(ret_ty.span);
@@ -415,11 +464,12 @@ impl<'a> AstValidator<'a> {
             let suggestion_span = header_span.shrink_to_hi().to(sig.decl.output.span());
             let padding = if header_span.is_empty() { "" } else { " " };
 
-            self.dcx().emit_err(errors::AbiCustomInvalidSignature {
+            self.dcx().emit_err(errors::AbiMustNotHaveParametersOrReturnType {
                 spans,
                 symbol: ident.name,
                 suggestion_span,
                 padding,
+                abi,
             });
         }
     }
@@ -1199,9 +1249,12 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 self.check_foreign_fn_bodyless(*ident, body.as_deref());
                 self.check_foreign_fn_headerless(sig.header);
                 self.check_foreign_item_ascii_only(*ident);
-                if self.extern_mod_abi == Some(ExternAbi::Custom) {
-                    self.check_custom_abi(FnCtxt::Foreign, ident, sig);
-                }
+                self.check_extern_fn_signature(
+                    self.extern_mod_abi.unwrap_or(ExternAbi::FALLBACK),
+                    FnCtxt::Foreign,
+                    ident,
+                    sig,
+                );
             }
             ForeignItemKind::TyAlias(box TyAlias {
                 defaultness,
@@ -1411,9 +1464,9 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
 
         if let FnKind::Fn(ctxt, _, fun) = fk
             && let Extern::Explicit(str_lit, _) = fun.sig.header.ext
-            && let Ok(ExternAbi::Custom) = ExternAbi::from_str(str_lit.symbol.as_str())
+            && let Ok(abi) = ExternAbi::from_str(str_lit.symbol.as_str())
         {
-            self.check_custom_abi(ctxt, &fun.ident, &fun.sig);
+            self.check_extern_fn_signature(abi, ctxt, &fun.ident, &fun.sig);
         }
 
         self.check_c_variadic_type(fk);
diff --git a/compiler/rustc_ast_passes/src/errors.rs b/compiler/rustc_ast_passes/src/errors.rs
index c437e62f4d3..d387a4a310e 100644
--- a/compiler/rustc_ast_passes/src/errors.rs
+++ b/compiler/rustc_ast_passes/src/errors.rs
@@ -1,5 +1,6 @@
 //! Errors emitted by ast_passes.
 
+use rustc_abi::ExternAbi;
 use rustc_ast::ParamKindOrd;
 use rustc_errors::codes::*;
 use rustc_errors::{Applicability, Diag, EmissionGuarantee, Subdiagnostic};
@@ -845,6 +846,7 @@ pub(crate) struct AbiCustomSafeForeignFunction {
 pub(crate) struct AbiCustomSafeFunction {
     #[primary_span]
     pub span: Span,
+    pub abi: ExternAbi,
 
     #[suggestion(
         ast_passes_suggestion,
@@ -856,10 +858,11 @@ pub(crate) struct AbiCustomSafeFunction {
 }
 
 #[derive(Diagnostic)]
-#[diag(ast_passes_abi_custom_coroutine)]
-pub(crate) struct AbiCustomCoroutine {
+#[diag(ast_passes_abi_cannot_be_coroutine)]
+pub(crate) struct AbiCannotBeCoroutine {
     #[primary_span]
     pub span: Span,
+    pub abi: ExternAbi,
 
     #[suggestion(
         ast_passes_suggestion,
@@ -872,11 +875,12 @@ pub(crate) struct AbiCustomCoroutine {
 }
 
 #[derive(Diagnostic)]
-#[diag(ast_passes_abi_custom_invalid_signature)]
+#[diag(ast_passes_abi_must_not_have_parameters_or_return_type)]
 #[note]
-pub(crate) struct AbiCustomInvalidSignature {
+pub(crate) struct AbiMustNotHaveParametersOrReturnType {
     #[primary_span]
     pub spans: Vec<Span>,
+    pub abi: ExternAbi,
 
     #[suggestion(
         ast_passes_suggestion,
@@ -888,3 +892,13 @@ pub(crate) struct AbiCustomInvalidSignature {
     pub symbol: Symbol,
     pub padding: &'static str,
 }
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_abi_must_not_have_return_type)]
+#[note]
+pub(crate) struct AbiMustNotHaveReturnType {
+    #[primary_span]
+    #[help]
+    pub span: Span,
+    pub abi: ExternAbi,
+}
diff --git a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
index f6b5ff404db..7651e8365a2 100644
--- a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
@@ -357,6 +357,10 @@ impl<'a> State<'a> {
                 self.word_nbsp("raw");
                 self.print_mutability(mutability, true);
             }
+            ast::BorrowKind::Pin => {
+                self.word_nbsp("pin");
+                self.print_mutability(mutability, true);
+            }
         }
         self.print_expr_cond_paren(
             expr,
diff --git a/compiler/rustc_attr_data_structures/src/attributes.rs b/compiler/rustc_attr_data_structures/src/attributes.rs
index ae3a1a41387..dc3598bcc36 100644
--- a/compiler/rustc_attr_data_structures/src/attributes.rs
+++ b/compiler/rustc_attr_data_structures/src/attributes.rs
@@ -212,6 +212,9 @@ pub enum AttributeKind {
         first_span: Span,
     },
 
+    /// Represents `#[const_continue]`.
+    ConstContinue(Span),
+
     /// Represents `#[rustc_const_stable]` and `#[rustc_const_unstable]`.
     ConstStability {
         stability: PartialConstStability,
@@ -231,6 +234,9 @@ pub enum AttributeKind {
     /// Represents `#[inline]` and `#[rustc_force_inline]`.
     Inline(InlineAttr, Span),
 
+    /// Represents `#[loop_match]`.
+    LoopMatch(Span),
+
     /// Represents `#[rustc_macro_transparency]`.
     MacroTransparency(Transparency),
 
@@ -268,5 +274,8 @@ pub enum AttributeKind {
         /// Span of the attribute.
         span: Span,
     },
+
+    /// Represents `#[track_caller]`
+    TrackCaller(Span),
     // tidy-alphabetical-end
 }
diff --git a/compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs b/compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs
index 0fa69c40154..eadf8657a0f 100644
--- a/compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs
@@ -1,7 +1,7 @@
 use rustc_attr_data_structures::{AttributeKind, OptimizeAttr};
 use rustc_feature::{AttributeTemplate, template};
 use rustc_session::parse::feature_err;
-use rustc_span::{Span, sym};
+use rustc_span::{Span, Symbol, sym};
 
 use super::{AcceptMapping, AttributeOrder, AttributeParser, OnDuplicate, SingleAttributeParser};
 use crate::context::{AcceptContext, FinalizeContext, Stage};
@@ -11,7 +11,7 @@ use crate::session_diagnostics::NakedFunctionIncompatibleAttribute;
 pub(crate) struct OptimizeParser;
 
 impl<S: Stage> SingleAttributeParser<S> for OptimizeParser {
-    const PATH: &[rustc_span::Symbol] = &[sym::optimize];
+    const PATH: &[Symbol] = &[sym::optimize];
     const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepLast;
     const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::WarnButFutureError;
     const TEMPLATE: AttributeTemplate = template!(List: "size|speed|none");
@@ -44,7 +44,7 @@ impl<S: Stage> SingleAttributeParser<S> for OptimizeParser {
 pub(crate) struct ColdParser;
 
 impl<S: Stage> SingleAttributeParser<S> for ColdParser {
-    const PATH: &[rustc_span::Symbol] = &[sym::cold];
+    const PATH: &[Symbol] = &[sym::cold];
     const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepLast;
     const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::Warn;
     const TEMPLATE: AttributeTemplate = template!(Word);
@@ -166,6 +166,24 @@ impl<S: Stage> AttributeParser<S> for NakedParser {
     }
 }
 
+pub(crate) struct TrackCallerParser;
+
+impl<S: Stage> SingleAttributeParser<S> for TrackCallerParser {
+    const PATH: &[Symbol] = &[sym::track_caller];
+    const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepLast;
+    const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::Warn;
+    const TEMPLATE: AttributeTemplate = template!(Word);
+
+    fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser<'_>) -> Option<AttributeKind> {
+        if let Err(span) = args.no_args() {
+            cx.expected_no_args(span);
+            return None;
+        }
+
+        Some(AttributeKind::TrackCaller(cx.attr_span))
+    }
+}
+
 pub(crate) struct NoMangleParser;
 
 impl<S: Stage> SingleAttributeParser<S> for NoMangleParser {
diff --git a/compiler/rustc_attr_parsing/src/attributes/loop_match.rs b/compiler/rustc_attr_parsing/src/attributes/loop_match.rs
new file mode 100644
index 00000000000..f6c7ac5e3a3
--- /dev/null
+++ b/compiler/rustc_attr_parsing/src/attributes/loop_match.rs
@@ -0,0 +1,31 @@
+use rustc_attr_data_structures::AttributeKind;
+use rustc_feature::{AttributeTemplate, template};
+use rustc_span::{Symbol, sym};
+
+use crate::attributes::{AttributeOrder, OnDuplicate, SingleAttributeParser};
+use crate::context::{AcceptContext, Stage};
+use crate::parser::ArgParser;
+
+pub(crate) struct LoopMatchParser;
+impl<S: Stage> SingleAttributeParser<S> for LoopMatchParser {
+    const PATH: &[Symbol] = &[sym::loop_match];
+    const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepFirst;
+    const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::Warn;
+    const TEMPLATE: AttributeTemplate = template!(Word);
+
+    fn convert(cx: &mut AcceptContext<'_, '_, S>, _args: &ArgParser<'_>) -> Option<AttributeKind> {
+        Some(AttributeKind::LoopMatch(cx.attr_span))
+    }
+}
+
+pub(crate) struct ConstContinueParser;
+impl<S: Stage> SingleAttributeParser<S> for ConstContinueParser {
+    const PATH: &[Symbol] = &[sym::const_continue];
+    const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepFirst;
+    const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::Warn;
+    const TEMPLATE: AttributeTemplate = template!(Word);
+
+    fn convert(cx: &mut AcceptContext<'_, '_, S>, _args: &ArgParser<'_>) -> Option<AttributeKind> {
+        Some(AttributeKind::ConstContinue(cx.attr_span))
+    }
+}
diff --git a/compiler/rustc_attr_parsing/src/attributes/mod.rs b/compiler/rustc_attr_parsing/src/attributes/mod.rs
index ac7e90fd902..d407669cb41 100644
--- a/compiler/rustc_attr_parsing/src/attributes/mod.rs
+++ b/compiler/rustc_attr_parsing/src/attributes/mod.rs
@@ -32,6 +32,7 @@ pub(crate) mod confusables;
 pub(crate) mod deprecation;
 pub(crate) mod inline;
 pub(crate) mod lint_helpers;
+pub(crate) mod loop_match;
 pub(crate) mod must_use;
 pub(crate) mod repr;
 pub(crate) mod semantics;
diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs
index 43b2d07dbe1..6ca5c64e0bc 100644
--- a/compiler/rustc_attr_parsing/src/context.rs
+++ b/compiler/rustc_attr_parsing/src/context.rs
@@ -15,11 +15,14 @@ use rustc_session::Session;
 use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span, Symbol, sym};
 
 use crate::attributes::allow_unstable::{AllowConstFnUnstableParser, AllowInternalUnstableParser};
-use crate::attributes::codegen_attrs::{ColdParser, NakedParser, NoMangleParser, OptimizeParser};
+use crate::attributes::codegen_attrs::{
+    ColdParser, NakedParser, NoMangleParser, OptimizeParser, TrackCallerParser,
+};
 use crate::attributes::confusables::ConfusablesParser;
 use crate::attributes::deprecation::DeprecationParser;
 use crate::attributes::inline::{InlineParser, RustcForceInlineParser};
 use crate::attributes::lint_helpers::{AsPtrParser, PubTransparentParser};
+use crate::attributes::loop_match::{ConstContinueParser, LoopMatchParser};
 use crate::attributes::must_use::MustUseParser;
 use crate::attributes::repr::{AlignParser, ReprParser};
 use crate::attributes::semantics::MayDangleParser;
@@ -111,9 +114,11 @@ attribute_parsers!(
         // tidy-alphabetical-start
         Single<AsPtrParser>,
         Single<ColdParser>,
+        Single<ConstContinueParser>,
         Single<ConstStabilityIndirectParser>,
         Single<DeprecationParser>,
         Single<InlineParser>,
+        Single<LoopMatchParser>,
         Single<MayDangleParser>,
         Single<MustUseParser>,
         Single<NoMangleParser>,
@@ -121,6 +126,7 @@ attribute_parsers!(
         Single<PubTransparentParser>,
         Single<RustcForceInlineParser>,
         Single<SkipDuringMethodDispatchParser>,
+        Single<TrackCallerParser>,
         Single<TransparencyParser>,
         // tidy-alphabetical-end
     ];
diff --git a/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs b/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
index 095c0df98ac..f9e52239d6f 100644
--- a/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
@@ -71,7 +71,6 @@ impl<'tcx> BorrowExplanation<'tcx> {
     ) {
         let tcx = cx.infcx.tcx;
         let body = cx.body;
-        let local_names = &cx.local_names;
 
         if let Some(span) = borrow_span {
             let def_id = body.source.def_id();
@@ -220,7 +219,7 @@ impl<'tcx> BorrowExplanation<'tcx> {
                     _ => ("destructor", format!("type `{}`", local_decl.ty)),
                 };
 
-                match local_names[dropped_local] {
+                match cx.local_name(dropped_local) {
                     Some(local_name) if !local_decl.from_compiler_desugaring() => {
                         let message = format!(
                             "{borrow_desc}borrow might be used here, when `{local_name}` is dropped \
@@ -670,10 +669,10 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
 
             Some(Cause::DropVar(local, location)) => {
                 let mut should_note_order = false;
-                if self.local_names[local].is_some()
+                if self.local_name(local).is_some()
                     && let Some((WriteKind::StorageDeadOrDrop, place)) = kind_place
                     && let Some(borrowed_local) = place.as_local()
-                    && self.local_names[borrowed_local].is_some()
+                    && self.local_name(borrowed_local).is_some()
                     && local != borrowed_local
                 {
                     should_note_order = true;
@@ -748,7 +747,7 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
                         Operand::Copy(place) | Operand::Move(place) => {
                             if let Some(l) = place.as_local() {
                                 let local_decl = &self.body.local_decls[l];
-                                if self.local_names[l].is_none() {
+                                if self.local_name(l).is_none() {
                                     local_decl.source_info.span
                                 } else {
                                     span
@@ -793,7 +792,7 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
                             Operand::Copy(place) | Operand::Move(place) => {
                                 if let Some(l) = place.as_local() {
                                     let local_decl = &self.body.local_decls[l];
-                                    if self.local_names[l].is_none() {
+                                    if self.local_name(l).is_none() {
                                         local_decl.source_info.span
                                     } else {
                                         span
diff --git a/compiler/rustc_borrowck/src/diagnostics/mod.rs b/compiler/rustc_borrowck/src/diagnostics/mod.rs
index 5e3f3ffa2ea..7b4e38969ee 100644
--- a/compiler/rustc_borrowck/src/diagnostics/mod.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/mod.rs
@@ -7,17 +7,17 @@ use rustc_data_structures::fx::FxIndexMap;
 use rustc_errors::{Applicability, Diag, EmissionGuarantee, MultiSpan, listify};
 use rustc_hir::def::{CtorKind, Namespace};
 use rustc_hir::{self as hir, CoroutineKind, LangItem};
-use rustc_index::IndexSlice;
+use rustc_index::{IndexSlice, IndexVec};
 use rustc_infer::infer::{BoundRegionConversionTime, NllRegionVariableOrigin};
 use rustc_infer::traits::SelectionError;
-use rustc_middle::bug;
 use rustc_middle::mir::{
     AggregateKind, CallSource, ConstOperand, ConstraintCategory, FakeReadCause, Local, LocalInfo,
     LocalKind, Location, Operand, Place, PlaceRef, PlaceTy, ProjectionElem, Rvalue, Statement,
-    StatementKind, Terminator, TerminatorKind, find_self_call,
+    StatementKind, Terminator, TerminatorKind, VarDebugInfoContents, find_self_call,
 };
 use rustc_middle::ty::print::Print;
 use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_middle::{bug, span_bug};
 use rustc_mir_dataflow::move_paths::{InitLocation, LookupResult, MoveOutIndex};
 use rustc_span::def_id::LocalDefId;
 use rustc_span::source_map::Spanned;
@@ -190,6 +190,36 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
     ) -> Option<&(PlaceRef<'tcx>, Diag<'infcx>)> {
         self.diags_buffer.buffered_move_errors.get(move_out_indices)
     }
+
+    /// Uses `body.var_debug_info` to find the symbol
+    fn local_name(&self, index: Local) -> Option<Symbol> {
+        *self.local_names().get(index)?
+    }
+
+    fn local_names(&self) -> &IndexSlice<Local, Option<Symbol>> {
+        self.local_names.get_or_init(|| {
+            let mut local_names = IndexVec::from_elem(None, &self.body.local_decls);
+            for var_debug_info in &self.body.var_debug_info {
+                if let VarDebugInfoContents::Place(place) = var_debug_info.value {
+                    if let Some(local) = place.as_local() {
+                        if let Some(prev_name) = local_names[local]
+                            && var_debug_info.name != prev_name
+                        {
+                            span_bug!(
+                                var_debug_info.source_info.span,
+                                "local {:?} has many names (`{}` vs `{}`)",
+                                local,
+                                prev_name,
+                                var_debug_info.name
+                            );
+                        }
+                        local_names[local] = Some(var_debug_info.name);
+                    }
+                }
+            }
+            local_names
+        })
+    }
 }
 
 impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
@@ -430,7 +460,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
     /// a name, or its name was generated by the compiler, then `Err` is returned
     fn append_local_to_string(&self, local: Local, buf: &mut String) -> Result<(), ()> {
         let decl = &self.body.local_decls[local];
-        match self.local_names[local] {
+        match self.local_name(local) {
             Some(name) if !decl.from_compiler_desugaring() => {
                 buf.push_str(name.as_str());
                 Ok(())
@@ -1500,4 +1530,9 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
             }
         }
     }
+
+    /// Skip over locals that begin with an underscore or have no name
+    pub(crate) fn local_excluded_from_unused_mut_lint(&self, index: Local) -> bool {
+        self.local_name(index).is_none_or(|name| name.as_str().starts_with('_'))
+    }
 }
diff --git a/compiler/rustc_borrowck/src/diagnostics/move_errors.rs b/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
index 0394a42ea9c..b21d348183f 100644
--- a/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
@@ -465,11 +465,15 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
 
         if let PlaceRef { local, projection: [] } = deref_base {
             let decl = &self.body.local_decls[local];
+            let local_name = self.local_name(local).map(|sym| format!("`{sym}`"));
             if decl.is_ref_for_guard() {
                 return self
                     .cannot_move_out_of(
                         span,
-                        &format!("`{}` in pattern guard", self.local_names[local].unwrap()),
+                        &format!(
+                            "{} in pattern guard",
+                            local_name.as_deref().unwrap_or("the place")
+                        ),
                     )
                     .with_note(
                         "variables bound in patterns cannot be moved from \
@@ -825,7 +829,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
             }
 
             if binds_to.len() == 1 {
-                let place_desc = &format!("`{}`", self.local_names[*local].unwrap());
+                let place_desc = self.local_name(*local).map(|sym| format!("`{sym}`"));
 
                 if let Some(expr) = self.find_expr(binding_span) {
                     self.suggest_cloning(err, bind_to.ty, expr, None);
@@ -834,7 +838,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                 err.subdiagnostic(crate::session_diagnostics::TypeNoCopy::Label {
                     is_partial_move: false,
                     ty: bind_to.ty,
-                    place: place_desc,
+                    place: place_desc.as_deref().unwrap_or("the place"),
                     span: binding_span,
                 });
             }
diff --git a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
index a5c9bad3ac2..fd8a2a6bc35 100644
--- a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
@@ -60,7 +60,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                 if access_place.as_local().is_some() {
                     reason = ", as it is not declared as mutable".to_string();
                 } else {
-                    let name = self.local_names[local].expect("immutable unnamed local");
+                    let name = self.local_name(local).expect("immutable unnamed local");
                     reason = format!(", as `{name}` is not declared as mutable");
                 }
             }
@@ -285,7 +285,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                     .body
                     .local_decls
                     .get(local)
-                    .is_some_and(|l| mut_borrow_of_mutable_ref(l, self.local_names[local])) =>
+                    .is_some_and(|l| mut_borrow_of_mutable_ref(l, self.local_name(local))) =>
             {
                 let decl = &self.body.local_decls[local];
                 err.span_label(span, format!("cannot {act}"));
@@ -481,7 +481,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                 let (pointer_sigil, pointer_desc) =
                     if local_decl.ty.is_ref() { ("&", "reference") } else { ("*const", "pointer") };
 
-                match self.local_names[local] {
+                match self.local_name(local) {
                     Some(name) if !local_decl.from_compiler_desugaring() => {
                         err.span_label(
                             span,
diff --git a/compiler/rustc_borrowck/src/diagnostics/region_errors.rs b/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
index 3bec07afa0f..d27e08573e0 100644
--- a/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
@@ -664,14 +664,14 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
         let fr_name_and_span = self.regioncx.get_var_name_and_span_for_region(
             self.infcx.tcx,
             self.body,
-            &self.local_names,
+            &self.local_names(),
             &self.upvars,
             errci.fr,
         );
         let outlived_fr_name_and_span = self.regioncx.get_var_name_and_span_for_region(
             self.infcx.tcx,
             self.body,
-            &self.local_names,
+            &self.local_names(),
             &self.upvars,
             errci.outlived_fr,
         );
diff --git a/compiler/rustc_borrowck/src/diagnostics/region_name.rs b/compiler/rustc_borrowck/src/diagnostics/region_name.rs
index 487f78058a8..1ad629ad167 100644
--- a/compiler/rustc_borrowck/src/diagnostics/region_name.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/region_name.rs
@@ -399,7 +399,7 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
             [implicit_inputs + argument_index];
         let (_, span) = self.regioncx.get_argument_name_and_span_for_region(
             self.body,
-            &self.local_names,
+            self.local_names(),
             argument_index,
         );
 
@@ -973,7 +973,7 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
         {
             let (arg_name, arg_span) = self.regioncx.get_argument_name_and_span_for_region(
                 self.body,
-                &self.local_names,
+                self.local_names(),
                 arg_index,
             );
             let region_name = self.synthesize_region_name();
diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs
index 4d85f109020..82b300dcb17 100644
--- a/compiler/rustc_borrowck/src/lib.rs
+++ b/compiler/rustc_borrowck/src/lib.rs
@@ -16,7 +16,7 @@
 // tidy-alphabetical-end
 
 use std::borrow::Cow;
-use std::cell::RefCell;
+use std::cell::{OnceCell, RefCell};
 use std::marker::PhantomData;
 use std::ops::{ControlFlow, Deref};
 
@@ -391,7 +391,7 @@ fn do_mir_borrowck<'tcx>(
             used_mut_upvars: SmallVec::new(),
             borrow_set: &borrow_set,
             upvars: &[],
-            local_names: IndexVec::from_elem(None, &promoted_body.local_decls),
+            local_names: OnceCell::from(IndexVec::from_elem(None, &promoted_body.local_decls)),
             region_names: RefCell::default(),
             next_region_name: RefCell::new(1),
             polonius_output: None,
@@ -414,26 +414,6 @@ fn do_mir_borrowck<'tcx>(
         promoted_mbcx.report_move_errors();
     }
 
-    let mut local_names = IndexVec::from_elem(None, &body.local_decls);
-    for var_debug_info in &body.var_debug_info {
-        if let VarDebugInfoContents::Place(place) = var_debug_info.value {
-            if let Some(local) = place.as_local() {
-                if let Some(prev_name) = local_names[local]
-                    && var_debug_info.name != prev_name
-                {
-                    span_bug!(
-                        var_debug_info.source_info.span,
-                        "local {:?} has many names (`{}` vs `{}`)",
-                        local,
-                        prev_name,
-                        var_debug_info.name
-                    );
-                }
-                local_names[local] = Some(var_debug_info.name);
-            }
-        }
-    }
-
     let mut mbcx = MirBorrowckCtxt {
         root_cx,
         infcx: &infcx,
@@ -450,7 +430,7 @@ fn do_mir_borrowck<'tcx>(
         used_mut_upvars: SmallVec::new(),
         borrow_set: &borrow_set,
         upvars: tcx.closure_captures(def),
-        local_names,
+        local_names: OnceCell::new(),
         region_names: RefCell::default(),
         next_region_name: RefCell::new(1),
         move_errors: Vec::new(),
@@ -682,7 +662,7 @@ struct MirBorrowckCtxt<'a, 'infcx, 'tcx> {
     upvars: &'tcx [&'tcx ty::CapturedPlace<'tcx>],
 
     /// Names of local (user) variables (extracted from `var_debug_info`).
-    local_names: IndexVec<Local, Option<Symbol>>,
+    local_names: OnceCell<IndexVec<Local, Option<Symbol>>>,
 
     /// Record the region names generated for each region in the given
     /// MIR def so that we can reuse them later in help/error messages.
@@ -2610,7 +2590,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> {
             };
 
             // Skip over locals that begin with an underscore or have no name
-            if self.local_names[local].is_none_or(|name| name.as_str().starts_with('_')) {
+            if self.local_excluded_from_unused_mut_lint(local) {
                 continue;
             }
 
diff --git a/compiler/rustc_builtin_macros/messages.ftl b/compiler/rustc_builtin_macros/messages.ftl
index c5d1f2ad2de..3594c7ec210 100644
--- a/compiler/rustc_builtin_macros/messages.ftl
+++ b/compiler/rustc_builtin_macros/messages.ftl
@@ -118,10 +118,6 @@ builtin_macros_concat_bytes_oob = numeric literal is out of bounds
 builtin_macros_concat_bytestr = cannot concatenate a byte string literal
 builtin_macros_concat_c_str_lit = cannot concatenate a C string literal
 
-builtin_macros_concat_idents_ident_args = `concat_idents!()` requires ident args
-
-builtin_macros_concat_idents_missing_args = `concat_idents!()` takes 1 or more arguments
-builtin_macros_concat_idents_missing_comma = `concat_idents!()` expecting comma
 builtin_macros_concat_missing_literal = expected a literal
     .note = only literals (like `"foo"`, `-42` and `3.14`) can be passed to `concat!()`
 
diff --git a/compiler/rustc_builtin_macros/src/concat_idents.rs b/compiler/rustc_builtin_macros/src/concat_idents.rs
deleted file mode 100644
index a721f5b84c5..00000000000
--- a/compiler/rustc_builtin_macros/src/concat_idents.rs
+++ /dev/null
@@ -1,71 +0,0 @@
-use rustc_ast::ptr::P;
-use rustc_ast::token::{self, Token};
-use rustc_ast::tokenstream::{TokenStream, TokenTree};
-use rustc_ast::{AttrVec, DUMMY_NODE_ID, Expr, ExprKind, Path, Ty, TyKind};
-use rustc_expand::base::{DummyResult, ExpandResult, ExtCtxt, MacResult, MacroExpanderResult};
-use rustc_span::{Ident, Span, Symbol};
-
-use crate::errors;
-
-pub(crate) fn expand_concat_idents<'cx>(
-    cx: &'cx mut ExtCtxt<'_>,
-    sp: Span,
-    tts: TokenStream,
-) -> MacroExpanderResult<'cx> {
-    if tts.is_empty() {
-        let guar = cx.dcx().emit_err(errors::ConcatIdentsMissingArgs { span: sp });
-        return ExpandResult::Ready(DummyResult::any(sp, guar));
-    }
-
-    let mut res_str = String::new();
-    for (i, e) in tts.iter().enumerate() {
-        if i & 1 == 1 {
-            match e {
-                TokenTree::Token(Token { kind: token::Comma, .. }, _) => {}
-                _ => {
-                    let guar = cx.dcx().emit_err(errors::ConcatIdentsMissingComma { span: sp });
-                    return ExpandResult::Ready(DummyResult::any(sp, guar));
-                }
-            }
-        } else {
-            if let TokenTree::Token(token, _) = e {
-                if let Some((ident, _)) = token.ident() {
-                    res_str.push_str(ident.name.as_str());
-                    continue;
-                }
-            }
-
-            let guar = cx.dcx().emit_err(errors::ConcatIdentsIdentArgs { span: sp });
-            return ExpandResult::Ready(DummyResult::any(sp, guar));
-        }
-    }
-
-    let ident = Ident::new(Symbol::intern(&res_str), cx.with_call_site_ctxt(sp));
-
-    struct ConcatIdentsResult {
-        ident: Ident,
-    }
-
-    impl MacResult for ConcatIdentsResult {
-        fn make_expr(self: Box<Self>) -> Option<P<Expr>> {
-            Some(P(Expr {
-                id: DUMMY_NODE_ID,
-                kind: ExprKind::Path(None, Path::from_ident(self.ident)),
-                span: self.ident.span,
-                attrs: AttrVec::new(),
-                tokens: None,
-            }))
-        }
-
-        fn make_ty(self: Box<Self>) -> Option<P<Ty>> {
-            Some(P(Ty {
-                id: DUMMY_NODE_ID,
-                kind: TyKind::Path(None, Path::from_ident(self.ident)),
-                span: self.ident.span,
-                tokens: None,
-            }))
-        }
-    }
-
-    ExpandResult::Ready(Box::new(ConcatIdentsResult { ident }))
-}
diff --git a/compiler/rustc_builtin_macros/src/errors.rs b/compiler/rustc_builtin_macros/src/errors.rs
index b7ecfd2285c..fe34cf5d6fa 100644
--- a/compiler/rustc_builtin_macros/src/errors.rs
+++ b/compiler/rustc_builtin_macros/src/errors.rs
@@ -300,27 +300,6 @@ pub(crate) struct ConcatBytesBadRepeat {
 }
 
 #[derive(Diagnostic)]
-#[diag(builtin_macros_concat_idents_missing_args)]
-pub(crate) struct ConcatIdentsMissingArgs {
-    #[primary_span]
-    pub(crate) span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(builtin_macros_concat_idents_missing_comma)]
-pub(crate) struct ConcatIdentsMissingComma {
-    #[primary_span]
-    pub(crate) span: Span,
-}
-
-#[derive(Diagnostic)]
-#[diag(builtin_macros_concat_idents_ident_args)]
-pub(crate) struct ConcatIdentsIdentArgs {
-    #[primary_span]
-    pub(crate) span: Span,
-}
-
-#[derive(Diagnostic)]
 #[diag(builtin_macros_bad_derive_target, code = E0774)]
 pub(crate) struct BadDeriveTarget {
     #[primary_span]
diff --git a/compiler/rustc_builtin_macros/src/lib.rs b/compiler/rustc_builtin_macros/src/lib.rs
index 9e7d0ec9e81..9b6dea21438 100644
--- a/compiler/rustc_builtin_macros/src/lib.rs
+++ b/compiler/rustc_builtin_macros/src/lib.rs
@@ -36,7 +36,6 @@ mod cfg_eval;
 mod compile_error;
 mod concat;
 mod concat_bytes;
-mod concat_idents;
 mod define_opaque;
 mod derive;
 mod deriving;
@@ -84,7 +83,6 @@ pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) {
         compile_error: compile_error::expand_compile_error,
         concat: concat::expand_concat,
         concat_bytes: concat_bytes::expand_concat_bytes,
-        concat_idents: concat_idents::expand_concat_idents,
         const_format_args: format::expand_format_args,
         core_panic: edition_panic::expand_panic,
         env: env::expand_env,
diff --git a/compiler/rustc_codegen_cranelift/Cargo.lock b/compiler/rustc_codegen_cranelift/Cargo.lock
index a906bec8b7e..b893a2be9a2 100644
--- a/compiler/rustc_codegen_cranelift/Cargo.lock
+++ b/compiler/rustc_codegen_cranelift/Cargo.lock
@@ -43,42 +43,42 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-assembler-x64"
-version = "0.120.0"
+version = "0.121.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ff8e35182c7372df00447cb90a04e584e032c42b9b9b6e8c50ddaaf0d7900d5"
+checksum = "f6f53499803b1607b6ee0ba0de4ba036e6da700c2e489fe8f9d0f683d0b84d31"
 dependencies = [
  "cranelift-assembler-x64-meta",
 ]
 
 [[package]]
 name = "cranelift-assembler-x64-meta"
-version = "0.120.0"
+version = "0.121.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14220f9c2698015c3b94dc6b84ae045c1c45509ddc406e43c6139252757fdb7a"
+checksum = "1aadaa5bc8430d0e7bb999459369bedd0e5816ad4a82a0e20748341c4e333eda"
 dependencies = [
  "cranelift-srcgen",
 ]
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.120.0"
+version = "0.121.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d372ef2777ceefd75829e1390211ac240e9196bc60699218f7ea2419038288ee"
+checksum = "2005fda2fc52a2dbce58229b4fb4483b70cbc806ba8ecc11b3f050c1a2d26cac"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-bitset"
-version = "0.120.0"
+version = "0.121.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56323783e423818fa89ce8078e90a3913d2a6e0810399bfce8ebd7ee87baa81f"
+checksum = "56935e02452ca1249d39ad5c45a96304d0b4300a158a391fd113451e0cd4483d"
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.120.0"
+version = "0.121.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74ffb780aab6186c6e9ba26519654b1ac55a09c0a866f6088a4efbbd84da68ed"
+checksum = "62612786bf00e10999f50217d6f455d02b31591155881a45a903d1a95d1a4043"
 dependencies = [
  "bumpalo",
  "cranelift-assembler-x64",
@@ -97,13 +97,14 @@ dependencies = [
  "serde",
  "smallvec",
  "target-lexicon",
+ "wasmtime-math",
 ]
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.120.0"
+version = "0.121.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c23ef13814d3b39c869650d5961128cbbecad83fbdff4e6836a03ecf6862d7ed"
+checksum = "07bae789df91ef236079733af9df11d852256c64af196f0bc6471ea0f5f301be"
 dependencies = [
  "cranelift-assembler-x64-meta",
  "cranelift-codegen-shared",
@@ -112,33 +113,33 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.120.0"
+version = "0.121.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9f623300657679f847803ce80811454bfff89cea4f6bf684be5c468d4a73631"
+checksum = "1be319616d36527782558a8312508757815f64deb19b094c7b8f4337229a9bc6"
 
 [[package]]
 name = "cranelift-control"
-version = "0.120.0"
+version = "0.121.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "31f4168af69989aa6b91fab46799ed4df6096f3209f4a6c8fb4358f49c60188f"
+checksum = "8810ee1ab5e9bd5cff4c0c8d240e2009cb5c2b79888fde1d5256d605712314b7"
 dependencies = [
  "arbitrary",
 ]
 
 [[package]]
 name = "cranelift-entity"
-version = "0.120.0"
+version = "0.121.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca6fa9bae1c8de26d71ac2162f069447610fd91e7780cb480ee0d76ac81eabb8"
+checksum = "086452c97cfbe116bf17dbe622dc5fdf2ea97299c7d4ce42460f284387c9928a"
 dependencies = [
  "cranelift-bitset",
 ]
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.120.0"
+version = "0.121.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8219205608aa0b0e6769b580284a7e055c7e0c323c1041cde7ca078add3e412"
+checksum = "4c27947010ab759330f252610c17a8cd64d123358be4f33164233d04fcd77b80"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -148,15 +149,15 @@ dependencies = [
 
 [[package]]
 name = "cranelift-isle"
-version = "0.120.0"
+version = "0.121.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "588d0c5964f10860b04043e55aab26d7f7a206b0fd4f10c5260e8aa5773832bd"
+checksum = "ec67bfb8bd55b1e9760eb9f5186dca8d81bd4d86110f8d5af01154a044c91802"
 
 [[package]]
 name = "cranelift-jit"
-version = "0.120.0"
+version = "0.121.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56bd917ddc524f84f4066f954062875bdfc0dffea068ee94e906d98de5ac7c33"
+checksum = "d67cdfc447f2abdb46bb30a6582cce189539c3c051c1d5330692376e1400edff"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -174,9 +175,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.120.0"
+version = "0.121.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68a03c057d8a992e06596c871341e446af43ff9224f941e5b8adea39137a5391"
+checksum = "e4597eaa52bca1ed111986c7a7f70cdbe192f83d271d627201365078e37b7e84"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -185,9 +186,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-native"
-version = "0.120.0"
+version = "0.121.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19ed3c94cb97b14f92b6a94a1d45ef8c851f6a2ad9114e5d91d233f7da638fed"
+checksum = "75a9b63edea46e013fce459c46e500462cb03a0490fdd9c18fe42b1dd7b93aa1"
 dependencies = [
  "cranelift-codegen",
  "libc",
@@ -196,9 +197,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.120.0"
+version = "0.121.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a64dacef362a69375a604f6636e5e9a174fb96dba3b273646fcd9fa85c1d0997"
+checksum = "ce706f0166d5b7f31693dff521e87cb9858e12adf22ffcde93c4a2826f8f04a9"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -211,9 +212,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-srcgen"
-version = "0.120.0"
+version = "0.121.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85256fac1519a7d25a040c1d850fba67478f3f021ad5fdf738ba4425ee862dbf"
+checksum = "7d5870e266df8237b56cc98b04f5739c228565c92dd629ec6c66efa87271a158"
 
 [[package]]
 name = "crc32fast"
@@ -289,6 +290,12 @@ dependencies = [
 ]
 
 [[package]]
+name = "libm"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de"
+
+[[package]]
 name = "log"
 version = "0.4.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -446,9 +453,9 @@ checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
 
 [[package]]
 name = "wasmtime-jit-icache-coherence"
-version = "33.0.0"
+version = "34.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "175e924dbc944c185808466d1e90b5a7feb610f3b9abdfe26f8ee25fd1086d1c"
+checksum = "2eedc0324e37cf39b049f4dca0c30997eaab49f09006d5f4c1994e64e7b7dba8"
 dependencies = [
  "anyhow",
  "cfg-if",
@@ -457,6 +464,15 @@ dependencies = [
 ]
 
 [[package]]
+name = "wasmtime-math"
+version = "34.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1cd35fae4cf51d2b4a9bd2ef04b0eb309fa1849cab6a6ab5ac27cbd054ea284d"
+dependencies = [
+ "libm",
+]
+
+[[package]]
 name = "windows-sys"
 version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/compiler/rustc_codegen_cranelift/Cargo.toml b/compiler/rustc_codegen_cranelift/Cargo.toml
index 94fcbd0a502..9066e4dbbb5 100644
--- a/compiler/rustc_codegen_cranelift/Cargo.toml
+++ b/compiler/rustc_codegen_cranelift/Cargo.toml
@@ -8,12 +8,12 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { version = "0.120.0", default-features = false, features = ["std", "timing", "unwind", "all-native-arch"] }
-cranelift-frontend = { version = "0.120.0" }
-cranelift-module = { version = "0.120.0" }
-cranelift-native = { version = "0.120.0" }
-cranelift-jit = { version = "0.120.0", optional = true }
-cranelift-object = { version = "0.120.0" }
+cranelift-codegen = { version = "0.121.0", default-features = false, features = ["std", "timing", "unwind", "all-native-arch"] }
+cranelift-frontend = { version = "0.121.0" }
+cranelift-module = { version = "0.121.0" }
+cranelift-native = { version = "0.121.0" }
+cranelift-jit = { version = "0.121.0", optional = true }
+cranelift-object = { version = "0.121.0" }
 target-lexicon = "0.13"
 gimli = { version = "0.31", default-features = false, features = ["write"] }
 object = { version = "0.36", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
@@ -24,12 +24,12 @@ smallvec = "1.8.1"
 
 [patch.crates-io]
 # Uncomment to use an unreleased version of cranelift
-#cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" }
-#cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" }
-#cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" }
-#cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" }
-#cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" }
-#cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" }
+#cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-34.0.0", version = "0.121.0" }
+#cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-34.0.0", version = "0.121.0" }
+#cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-34.0.0", version = "0.121.0" }
+#cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-34.0.0", version = "0.121.0" }
+#cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-34.0.0", version = "0.121.0" }
+#cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-34.0.0", version = "0.121.0" }
 
 # Uncomment to use local checkout of cranelift
 #cranelift-codegen = { path = "../wasmtime/cranelift/codegen" }
diff --git a/compiler/rustc_codegen_cranelift/build_system/abi_cafe.rs b/compiler/rustc_codegen_cranelift/build_system/abi_cafe.rs
index 674acfbd309..43025137bc6 100644
--- a/compiler/rustc_codegen_cranelift/build_system/abi_cafe.rs
+++ b/compiler/rustc_codegen_cranelift/build_system/abi_cafe.rs
@@ -6,8 +6,8 @@ use crate::{CodegenBackend, SysrootKind, build_sysroot};
 static ABI_CAFE_REPO: GitRepo = GitRepo::github(
     "Gankra",
     "abi-cafe",
-    "f1220cfd13b57f5c0082c26529163865ee25e115",
-    "fe93a9acd461425d",
+    "94d38030419eb00a1ba80e5e2b4d763dcee58db4",
+    "6efb4457893c8670",
     "abi-cafe",
 );
 
@@ -46,6 +46,10 @@ pub(crate) fn run(
     let mut cmd = ABI_CAFE.run(bootstrap_host_compiler, dirs);
     cmd.arg("--");
 
+    cmd.arg("--debug");
+
+    cmd.arg("--rules").arg(dirs.source_dir.join("scripts/abi-cafe-rules.toml"));
+
     // stdcall, vectorcall and such don't work yet
     cmd.arg("--conventions").arg("c").arg("--conventions").arg("rust");
 
diff --git a/compiler/rustc_codegen_cranelift/patches/0002-abi-cafe-Disable-broken-tests.patch b/compiler/rustc_codegen_cranelift/patches/0002-abi-cafe-Disable-broken-tests.patch
deleted file mode 100644
index 01b6a990b72..00000000000
--- a/compiler/rustc_codegen_cranelift/patches/0002-abi-cafe-Disable-broken-tests.patch
+++ /dev/null
@@ -1,69 +0,0 @@
-From 236df390f3bc4ed69c26f4d51d584bea246da886 Mon Sep 17 00:00:00 2001
-From: bjorn3 <17426603+bjorn3@users.noreply.github.com>
-Date: Tue, 9 Jul 2024 11:25:14 +0000
-Subject: [PATCH] Disable broken tests
-
----
- src/report.rs | 36 ++++++++++++++++++++++++++++++++++++
- 1 file changed, 36 insertions(+)
-
-diff --git a/src/toolchains/rust.rs b/src/toolchains/rust.rs
-index 0c50f7a..bfde2b1 100644
---- a/src/toolchains/rust.rs
-+++ b/src/toolchains/rust.rs
-@@ -83,6 +83,7 @@ impl Toolchain for RustcToolchain {
-             .arg(out_dir)
-             .arg("--target")
-             .arg(built_info::TARGET)
-+            .arg("-g")
-             .arg(format!("-Cmetadata={lib_name}"))
-             .arg(src_path);
-         if let Some(codegen_backend) = &self.codegen_backend {
-diff --git a/src/report.rs b/src/report.rs
-index 958ab43..dcf1044 100644
---- a/src/report.rs
-+++ b/src/report.rs
-@@ -48,6 +48,40 @@ pub fn get_test_rules(test: &TestKey, caller: &dyn Toolchain, callee: &dyn Toolc
-     //
-     // THIS AREA RESERVED FOR VENDORS TO APPLY PATCHES
- 
-+    if cfg!(all(target_arch = "aarch64", target_os = "linux")) {
-+        if test.test == "F32Array" && test.options.convention == CallingConvention::C {
-+            result.check = Busted(Check);
-+        }
-+    }
-+
-+    if cfg!(all(target_arch = "aarch64", target_os = "macos")) {
-+        if test.test == "SingleVariantUnion" && test.options.convention == CallingConvention::C && test.options.repr == LangRepr::C {
-+            result.check = Busted(Check);
-+        }
-+
-+        if test.test == "OptionU128" && test.caller == "rustc" && test.options.convention == CallingConvention::Rust && test.options.repr == LangRepr::C {
-+            result.check = Busted(Run);
-+        }
-+
-+        if test.test == "OptionU128" && test.caller == "cgclif" && test.options.convention == CallingConvention::Rust && test.options.repr == LangRepr::C {
-+            result.check = Busted(Check);
-+        }
-+    }
-+
-+    if cfg!(all(target_arch = "x86_64", windows)) {
-+        if test.test == "simple" && test.options.convention == CallingConvention::Rust {
-+            result.check = Busted(Check);
-+        }
-+
-+        if test.test == "simple" && test.options.convention == CallingConvention::Rust && test.caller == "rustc" {
-+            result.check = Busted(Run);
-+        }
-+    }
-+
-+    if test.test == "f16" || test.test == "f128" {
-+        result.run = Skip;
-+    }
-+
-     // END OF VENDOR RESERVED AREA
-     //
-     //
--- 
-2.34.1
-
diff --git a/compiler/rustc_codegen_cranelift/rust-toolchain b/compiler/rustc_codegen_cranelift/rust-toolchain
index af4bd6dc6b8..150bb562f74 100644
--- a/compiler/rustc_codegen_cranelift/rust-toolchain
+++ b/compiler/rustc_codegen_cranelift/rust-toolchain
@@ -1,4 +1,4 @@
 [toolchain]
-channel = "nightly-2025-05-25"
+channel = "nightly-2025-06-24"
 components = ["rust-src", "rustc-dev", "llvm-tools"]
 profile = "minimal"
diff --git a/compiler/rustc_codegen_cranelift/scripts/abi-cafe-rules.toml b/compiler/rustc_codegen_cranelift/scripts/abi-cafe-rules.toml
new file mode 100644
index 00000000000..54f9445c8e5
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/scripts/abi-cafe-rules.toml
@@ -0,0 +1,17 @@
+[target.'cfg(all(target_arch = "aarch64", target_os = "linux"))']
+'F32Array::conv_c'.busted = "check"
+
+[target.'cfg(all(target_arch = "aarch64", target_os = "macos"))']
+'SingleVariantUnion::conv_c::repr_c'.busted = "check"
+'OptionU128::conv_rust::repr_c::rustc_caller'.busted = "run"
+'OptionU128::conv_rust::repr_c::cgclif_caller'.busted = "check"
+
+[target.'cfg(all(target_arch = "x86_64", windows))']
+'simple::conv_rust'.busted = "check"
+'simple::conv_rust::rustc_caller'.busted = "run"
+
+[target.'*'.'f16']
+run = "skip"
+
+[target.'*'.'f128']
+run = "skip"
diff --git a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
index 32c71f433b0..7e356b4b462 100755
--- a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
@@ -151,20 +151,6 @@ rm tests/ui/process/process-panic-after-fork.rs # same
 cp ../dist/bin/rustdoc-clif ../dist/bin/rustdoc # some tests expect bin/rustdoc to exist
 
 cat <<EOF | git apply -
-diff --git a/tests/run-make/linker-warning/rmake.rs b/tests/run-make/linker-warning/rmake.rs
-index 30387af428c..f7895b12961 100644
---- a/tests/run-make/linker-warning/rmake.rs
-+++ b/tests/run-make/linker-warning/rmake.rs
-@@ -57,7 +57,8 @@ fn main() {
-             .actual_text("(linker error)", out.stderr())
--            .normalize(r#"/rustc[^/]*/"#, "/rustc/")
-+            .normalize(r#"/tmp/rustc[^/]*/"#, "/tmp/rustc/")
-+            .normalize("libpanic_abort", "libpanic_unwind")
-             .normalize(
-                 regex::escape(run_make_support::build_root().to_str().unwrap()),
-                 "/build-root",
-             )
-             .normalize(r#""[^"]*\/symbols.o""#, "\\"/symbols.o\\"")
 diff --git a/src/tools/compiletest/src/runtest/run_make.rs b/src/tools/compiletest/src/runtest/run_make.rs
 index 073116933bd..c3e4578204d 100644
 --- a/src/tools/compiletest/src/runtest/run_make.rs
diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs
index c8527c3a57d..3a62cd52a9d 100644
--- a/compiler/rustc_codegen_cranelift/src/constant.rs
+++ b/compiler/rustc_codegen_cranelift/src/constant.rs
@@ -228,7 +228,7 @@ fn pointer_for_allocation<'tcx>(
     crate::pointer::Pointer::new(global_ptr)
 }
 
-pub(crate) fn data_id_for_alloc_id(
+fn data_id_for_alloc_id(
     cx: &mut ConstantCx,
     module: &mut dyn Module,
     alloc_id: AllocId,
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
index 615f6c47d90..37fbe4be1b0 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
@@ -202,9 +202,10 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             };
             let x = codegen_operand(fx, &x.node);
             let y = codegen_operand(fx, &y.node);
-            let kind = match &kind.node {
-                Operand::Constant(const_) => crate::constant::eval_mir_constant(fx, const_).0,
-                Operand::Copy(_) | Operand::Move(_) => unreachable!("{kind:?}"),
+            let kind = if let Some(const_) = kind.node.constant() {
+                crate::constant::eval_mir_constant(fx, const_).0
+            } else {
+                unreachable!("{kind:?}")
             };
 
             let flt_cc = match kind
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
index 46a441488fa..68ff0b622c8 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
@@ -205,9 +205,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             // Find a way to reuse `immediate_const_vector` from `codegen_ssa` instead.
             let indexes = {
                 use rustc_middle::mir::interpret::*;
-                let idx_const = match &idx.node {
-                    Operand::Constant(const_) => crate::constant::eval_mir_constant(fx, const_).0,
-                    Operand::Copy(_) | Operand::Move(_) => unreachable!("{idx:?}"),
+                let idx_const = if let Some(const_) = idx.node.constant() {
+                    crate::constant::eval_mir_constant(fx, const_).0
+                } else {
+                    unreachable!("{idx:?}")
                 };
 
                 let idx_bytes = match idx_const {
diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs
index 8882ba359b7..4a2425967e4 100644
--- a/compiler/rustc_codegen_ssa/src/back/link.rs
+++ b/compiler/rustc_codegen_ssa/src/back/link.rs
@@ -865,7 +865,7 @@ fn link_natively(
                     command: cmd,
                     escaped_output,
                     verbose: sess.opts.verbose,
-                    sysroot_dir: sess.sysroot.clone(),
+                    sysroot_dir: sess.opts.sysroot.path().to_owned(),
                 };
                 sess.dcx().emit_err(err);
                 // If MSVC's `link.exe` was expected but the return code
@@ -1249,10 +1249,10 @@ fn link_sanitizer_runtime(
         if path.exists() {
             sess.target_tlib_path.dir.clone()
         } else {
-            let default_sysroot = filesearch::get_or_default_sysroot();
-            let default_tlib =
-                filesearch::make_target_lib_path(&default_sysroot, sess.opts.target_triple.tuple());
-            default_tlib
+            filesearch::make_target_lib_path(
+                &sess.opts.sysroot.default,
+                sess.opts.target_triple.tuple(),
+            )
         }
     }
 
@@ -1758,7 +1758,7 @@ fn detect_self_contained_mingw(sess: &Session, linker: &Path) -> bool {
     for dir in env::split_paths(&env::var_os("PATH").unwrap_or_default()) {
         let full_path = dir.join(&linker_with_extension);
         // If linker comes from sysroot assume self-contained mode
-        if full_path.is_file() && !full_path.starts_with(&sess.sysroot) {
+        if full_path.is_file() && !full_path.starts_with(sess.opts.sysroot.path()) {
             return false;
         }
     }
diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs
index 8fc83908efb..cd08c0fc30f 100644
--- a/compiler/rustc_codegen_ssa/src/back/linker.rs
+++ b/compiler/rustc_codegen_ssa/src/back/linker.rs
@@ -337,7 +337,12 @@ pub(crate) trait Linker {
     fn debuginfo(&mut self, strip: Strip, natvis_debugger_visualizers: &[PathBuf]);
     fn no_crt_objects(&mut self);
     fn no_default_libraries(&mut self);
-    fn export_symbols(&mut self, tmpdir: &Path, crate_type: CrateType, symbols: &[String]);
+    fn export_symbols(
+        &mut self,
+        tmpdir: &Path,
+        crate_type: CrateType,
+        symbols: &[(String, SymbolExportKind)],
+    );
     fn subsystem(&mut self, subsystem: &str);
     fn linker_plugin_lto(&mut self);
     fn add_eh_frame_header(&mut self) {}
@@ -770,7 +775,12 @@ impl<'a> Linker for GccLinker<'a> {
         }
     }
 
-    fn export_symbols(&mut self, tmpdir: &Path, crate_type: CrateType, symbols: &[String]) {
+    fn export_symbols(
+        &mut self,
+        tmpdir: &Path,
+        crate_type: CrateType,
+        symbols: &[(String, SymbolExportKind)],
+    ) {
         // Symbol visibility in object files typically takes care of this.
         if crate_type == CrateType::Executable {
             let should_export_executable_symbols =
@@ -799,7 +809,7 @@ impl<'a> Linker for GccLinker<'a> {
             // Write a plain, newline-separated list of symbols
             let res: io::Result<()> = try {
                 let mut f = File::create_buffered(&path)?;
-                for sym in symbols {
+                for (sym, _) in symbols {
                     debug!("  _{sym}");
                     writeln!(f, "_{sym}")?;
                 }
@@ -814,11 +824,12 @@ impl<'a> Linker for GccLinker<'a> {
                 // .def file similar to MSVC one but without LIBRARY section
                 // because LD doesn't like when it's empty
                 writeln!(f, "EXPORTS")?;
-                for symbol in symbols {
+                for (symbol, kind) in symbols {
+                    let kind_marker = if *kind == SymbolExportKind::Data { " DATA" } else { "" };
                     debug!("  _{symbol}");
                     // Quote the name in case it's reserved by linker in some way
                     // (this accounts for names with dots in particular).
-                    writeln!(f, "  \"{symbol}\"")?;
+                    writeln!(f, "  \"{symbol}\"{kind_marker}")?;
                 }
             };
             if let Err(error) = res {
@@ -831,7 +842,7 @@ impl<'a> Linker for GccLinker<'a> {
                 writeln!(f, "{{")?;
                 if !symbols.is_empty() {
                     writeln!(f, "  global:")?;
-                    for sym in symbols {
+                    for (sym, _) in symbols {
                         debug!("    {sym};");
                         writeln!(f, "    {sym};")?;
                     }
@@ -1059,7 +1070,7 @@ impl<'a> Linker for MsvcLinker<'a> {
         self.link_arg("/PDBALTPATH:%_PDB%");
 
         // This will cause the Microsoft linker to embed .natvis info into the PDB file
-        let natvis_dir_path = self.sess.sysroot.join("lib\\rustlib\\etc");
+        let natvis_dir_path = self.sess.opts.sysroot.path().join("lib\\rustlib\\etc");
         if let Ok(natvis_dir) = fs::read_dir(&natvis_dir_path) {
             for entry in natvis_dir {
                 match entry {
@@ -1098,7 +1109,12 @@ impl<'a> Linker for MsvcLinker<'a> {
     // crates. Upstream rlibs may be linked statically to this dynamic library,
     // in which case they may continue to transitively be used and hence need
     // their symbols exported.
-    fn export_symbols(&mut self, tmpdir: &Path, crate_type: CrateType, symbols: &[String]) {
+    fn export_symbols(
+        &mut self,
+        tmpdir: &Path,
+        crate_type: CrateType,
+        symbols: &[(String, SymbolExportKind)],
+    ) {
         // Symbol visibility takes care of this typically
         if crate_type == CrateType::Executable {
             let should_export_executable_symbols =
@@ -1116,9 +1132,10 @@ impl<'a> Linker for MsvcLinker<'a> {
             // straight to exports.
             writeln!(f, "LIBRARY")?;
             writeln!(f, "EXPORTS")?;
-            for symbol in symbols {
+            for (symbol, kind) in symbols {
+                let kind_marker = if *kind == SymbolExportKind::Data { " DATA" } else { "" };
                 debug!("  _{symbol}");
-                writeln!(f, "  {symbol}")?;
+                writeln!(f, "  {symbol}{kind_marker}")?;
             }
         };
         if let Err(error) = res {
@@ -1259,14 +1276,19 @@ impl<'a> Linker for EmLinker<'a> {
         self.cc_arg("-nodefaultlibs");
     }
 
-    fn export_symbols(&mut self, _tmpdir: &Path, _crate_type: CrateType, symbols: &[String]) {
+    fn export_symbols(
+        &mut self,
+        _tmpdir: &Path,
+        _crate_type: CrateType,
+        symbols: &[(String, SymbolExportKind)],
+    ) {
         debug!("EXPORTED SYMBOLS:");
 
         self.cc_arg("-s");
 
         let mut arg = OsString::from("EXPORTED_FUNCTIONS=");
         let encoded = serde_json::to_string(
-            &symbols.iter().map(|sym| "_".to_owned() + sym).collect::<Vec<_>>(),
+            &symbols.iter().map(|(sym, _)| "_".to_owned() + sym).collect::<Vec<_>>(),
         )
         .unwrap();
         debug!("{encoded}");
@@ -1428,8 +1450,13 @@ impl<'a> Linker for WasmLd<'a> {
 
     fn no_default_libraries(&mut self) {}
 
-    fn export_symbols(&mut self, _tmpdir: &Path, _crate_type: CrateType, symbols: &[String]) {
-        for sym in symbols {
+    fn export_symbols(
+        &mut self,
+        _tmpdir: &Path,
+        _crate_type: CrateType,
+        symbols: &[(String, SymbolExportKind)],
+    ) {
+        for (sym, _) in symbols {
             self.link_args(&["--export", sym]);
         }
 
@@ -1563,7 +1590,7 @@ impl<'a> Linker for L4Bender<'a> {
         self.cc_arg("-nostdlib");
     }
 
-    fn export_symbols(&mut self, _: &Path, _: CrateType, _: &[String]) {
+    fn export_symbols(&mut self, _: &Path, _: CrateType, _: &[(String, SymbolExportKind)]) {
         // ToDo, not implemented, copy from GCC
         self.sess.dcx().emit_warn(errors::L4BenderExportingSymbolsUnimplemented);
     }
@@ -1720,12 +1747,17 @@ impl<'a> Linker for AixLinker<'a> {
 
     fn no_default_libraries(&mut self) {}
 
-    fn export_symbols(&mut self, tmpdir: &Path, _crate_type: CrateType, symbols: &[String]) {
+    fn export_symbols(
+        &mut self,
+        tmpdir: &Path,
+        _crate_type: CrateType,
+        symbols: &[(String, SymbolExportKind)],
+    ) {
         let path = tmpdir.join("list.exp");
         let res: io::Result<()> = try {
             let mut f = File::create_buffered(&path)?;
             // FIXME: use llvm-nm to generate export list.
-            for symbol in symbols {
+            for (symbol, _) in symbols {
                 debug!("  _{symbol}");
                 writeln!(f, "  {symbol}")?;
             }
@@ -1769,9 +1801,23 @@ fn for_each_exported_symbols_include_dep<'tcx>(
     }
 }
 
-pub(crate) fn exported_symbols(tcx: TyCtxt<'_>, crate_type: CrateType) -> Vec<String> {
+pub(crate) fn exported_symbols(
+    tcx: TyCtxt<'_>,
+    crate_type: CrateType,
+) -> Vec<(String, SymbolExportKind)> {
     if let Some(ref exports) = tcx.sess.target.override_export_symbols {
-        return exports.iter().map(ToString::to_string).collect();
+        return exports
+            .iter()
+            .map(|name| {
+                (
+                    name.to_string(),
+                    // FIXME use the correct export kind for this symbol. override_export_symbols
+                    // can't directly specify the SymbolExportKind as it is defined in rustc_middle
+                    // which rustc_target can't depend on.
+                    SymbolExportKind::Text,
+                )
+            })
+            .collect();
     }
 
     if let CrateType::ProcMacro = crate_type {
@@ -1781,7 +1827,10 @@ pub(crate) fn exported_symbols(tcx: TyCtxt<'_>, crate_type: CrateType) -> Vec<St
     }
 }
 
-fn exported_symbols_for_non_proc_macro(tcx: TyCtxt<'_>, crate_type: CrateType) -> Vec<String> {
+fn exported_symbols_for_non_proc_macro(
+    tcx: TyCtxt<'_>,
+    crate_type: CrateType,
+) -> Vec<(String, SymbolExportKind)> {
     let mut symbols = Vec::new();
     let export_threshold = symbol_export::crates_export_threshold(&[crate_type]);
     for_each_exported_symbols_include_dep(tcx, crate_type, |symbol, info, cnum| {
@@ -1789,8 +1838,9 @@ fn exported_symbols_for_non_proc_macro(tcx: TyCtxt<'_>, crate_type: CrateType) -
         // from any cdylib. The latter doesn't work anyway as we use hidden visibility for
         // compiler-builtins. Most linkers silently ignore it, but ld64 gives a warning.
         if info.level.is_below_threshold(export_threshold) && !tcx.is_compiler_builtins(cnum) {
-            symbols.push(symbol_export::exporting_symbol_name_for_instance_in_crate(
-                tcx, symbol, cnum,
+            symbols.push((
+                symbol_export::exporting_symbol_name_for_instance_in_crate(tcx, symbol, cnum),
+                info.kind,
             ));
             symbol_export::extend_exported_symbols(&mut symbols, tcx, symbol, cnum);
         }
@@ -1799,7 +1849,7 @@ fn exported_symbols_for_non_proc_macro(tcx: TyCtxt<'_>, crate_type: CrateType) -
     symbols
 }
 
-fn exported_symbols_for_proc_macro_crate(tcx: TyCtxt<'_>) -> Vec<String> {
+fn exported_symbols_for_proc_macro_crate(tcx: TyCtxt<'_>) -> Vec<(String, SymbolExportKind)> {
     // `exported_symbols` will be empty when !should_codegen.
     if !tcx.sess.opts.output_types.should_codegen() {
         return Vec::new();
@@ -1809,7 +1859,10 @@ fn exported_symbols_for_proc_macro_crate(tcx: TyCtxt<'_>) -> Vec<String> {
     let proc_macro_decls_name = tcx.sess.generate_proc_macro_decls_symbol(stable_crate_id);
     let metadata_symbol_name = exported_symbols::metadata_symbol_name(tcx);
 
-    vec![proc_macro_decls_name, metadata_symbol_name]
+    vec![
+        (proc_macro_decls_name, SymbolExportKind::Data),
+        (metadata_symbol_name, SymbolExportKind::Data),
+    ]
 }
 
 pub(crate) fn linked_symbols(
@@ -1831,7 +1884,9 @@ pub(crate) fn linked_symbols(
             || info.used
         {
             symbols.push((
-                symbol_export::linking_symbol_name_for_instance_in_crate(tcx, symbol, cnum),
+                symbol_export::linking_symbol_name_for_instance_in_crate(
+                    tcx, symbol, info.kind, cnum,
+                ),
                 info.kind,
             ));
         }
@@ -1906,7 +1961,13 @@ impl<'a> Linker for PtxLinker<'a> {
 
     fn ehcont_guard(&mut self) {}
 
-    fn export_symbols(&mut self, _tmpdir: &Path, _crate_type: CrateType, _symbols: &[String]) {}
+    fn export_symbols(
+        &mut self,
+        _tmpdir: &Path,
+        _crate_type: CrateType,
+        _symbols: &[(String, SymbolExportKind)],
+    ) {
+    }
 
     fn subsystem(&mut self, _subsystem: &str) {}
 
@@ -1975,10 +2036,15 @@ impl<'a> Linker for LlbcLinker<'a> {
 
     fn ehcont_guard(&mut self) {}
 
-    fn export_symbols(&mut self, _tmpdir: &Path, _crate_type: CrateType, symbols: &[String]) {
+    fn export_symbols(
+        &mut self,
+        _tmpdir: &Path,
+        _crate_type: CrateType,
+        symbols: &[(String, SymbolExportKind)],
+    ) {
         match _crate_type {
             CrateType::Cdylib => {
-                for sym in symbols {
+                for (sym, _) in symbols {
                     self.link_args(&["--export-symbol", sym]);
                 }
             }
@@ -2052,11 +2118,16 @@ impl<'a> Linker for BpfLinker<'a> {
 
     fn ehcont_guard(&mut self) {}
 
-    fn export_symbols(&mut self, tmpdir: &Path, _crate_type: CrateType, symbols: &[String]) {
+    fn export_symbols(
+        &mut self,
+        tmpdir: &Path,
+        _crate_type: CrateType,
+        symbols: &[(String, SymbolExportKind)],
+    ) {
         let path = tmpdir.join("symbols");
         let res: io::Result<()> = try {
             let mut f = File::create_buffered(&path)?;
-            for sym in symbols {
+            for (sym, _) in symbols {
                 writeln!(f, "{sym}")?;
             }
         };
diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
index d0b6c7470fb..19c005d418e 100644
--- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
+++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
@@ -680,6 +680,7 @@ fn calling_convention_for_symbol<'tcx>(
 pub(crate) fn linking_symbol_name_for_instance_in_crate<'tcx>(
     tcx: TyCtxt<'tcx>,
     symbol: ExportedSymbol<'tcx>,
+    export_kind: SymbolExportKind,
     instantiating_crate: CrateNum,
 ) -> String {
     let mut undecorated = symbol_name_for_instance_in_crate(tcx, symbol, instantiating_crate);
@@ -700,8 +701,9 @@ pub(crate) fn linking_symbol_name_for_instance_in_crate<'tcx>(
     let prefix = match &target.arch[..] {
         "x86" => Some('_'),
         "x86_64" => None,
-        "arm64ec" => Some('#'),
-        // Only x86/64 use symbol decorations.
+        // Only functions are decorated for arm64ec.
+        "arm64ec" if export_kind == SymbolExportKind::Text => Some('#'),
+        // Only x86/64 and arm64ec use symbol decorations.
         _ => return undecorated,
     };
 
@@ -741,7 +743,7 @@ pub(crate) fn exporting_symbol_name_for_instance_in_crate<'tcx>(
 /// Add it to the symbols list for all kernel functions, so that it is exported in the linked
 /// object.
 pub(crate) fn extend_exported_symbols<'tcx>(
-    symbols: &mut Vec<String>,
+    symbols: &mut Vec<(String, SymbolExportKind)>,
     tcx: TyCtxt<'tcx>,
     symbol: ExportedSymbol<'tcx>,
     instantiating_crate: CrateNum,
@@ -755,7 +757,9 @@ pub(crate) fn extend_exported_symbols<'tcx>(
     let undecorated = symbol_name_for_instance_in_crate(tcx, symbol, instantiating_crate);
 
     // Add the symbol for the kernel descriptor (with .kd suffix)
-    symbols.push(format!("{undecorated}.kd"));
+    // Per https://llvm.org/docs/AMDGPUUsage.html#symbols these will always be `STT_OBJECT` so
+    // export as data.
+    symbols.push((format!("{undecorated}.kd"), SymbolExportKind::Data));
 }
 
 fn maybe_emutls_symbol_name<'tcx>(
diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs
index a3d6c73ba85..cc90271cd0c 100644
--- a/compiler/rustc_codegen_ssa/src/base.rs
+++ b/compiler/rustc_codegen_ssa/src/base.rs
@@ -12,9 +12,9 @@ use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
 use rustc_data_structures::profiling::{get_resident_set_size, print_time_passes_entry};
 use rustc_data_structures::sync::{IntoDynSyncSend, par_map};
 use rustc_data_structures::unord::UnordMap;
-use rustc_hir::ItemId;
 use rustc_hir::def_id::{DefId, LOCAL_CRATE};
 use rustc_hir::lang_items::LangItem;
+use rustc_hir::{ItemId, Target};
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
 use rustc_middle::middle::debugger_visualizer::{DebuggerVisualizerFile, DebuggerVisualizerType};
 use rustc_middle::middle::exported_symbols::{self, SymbolExportKind};
@@ -1003,21 +1003,35 @@ impl CrateInfo {
         // by the compiler, but that's ok because all this stuff is unstable anyway.
         let target = &tcx.sess.target;
         if !are_upstream_rust_objects_already_included(tcx.sess) {
-            let missing_weak_lang_items: FxIndexSet<Symbol> = info
+            let add_prefix = match (target.is_like_windows, target.arch.as_ref()) {
+                (true, "x86") => |name: String, _: SymbolExportKind| format!("_{name}"),
+                (true, "arm64ec") => {
+                    // Only functions are decorated for arm64ec.
+                    |name: String, export_kind: SymbolExportKind| match export_kind {
+                        SymbolExportKind::Text => format!("#{name}"),
+                        _ => name,
+                    }
+                }
+                _ => |name: String, _: SymbolExportKind| name,
+            };
+            let missing_weak_lang_items: FxIndexSet<(Symbol, SymbolExportKind)> = info
                 .used_crates
                 .iter()
                 .flat_map(|&cnum| tcx.missing_lang_items(cnum))
                 .filter(|l| l.is_weak())
                 .filter_map(|&l| {
                     let name = l.link_name()?;
-                    lang_items::required(tcx, l).then_some(name)
+                    let export_kind = match l.target() {
+                        Target::Fn => SymbolExportKind::Text,
+                        Target::Static => SymbolExportKind::Data,
+                        _ => bug!(
+                            "Don't know what the export kind is for lang item of kind {:?}",
+                            l.target()
+                        ),
+                    };
+                    lang_items::required(tcx, l).then_some((name, export_kind))
                 })
                 .collect();
-            let prefix = match (target.is_like_windows, target.arch.as_ref()) {
-                (true, "x86") => "_",
-                (true, "arm64ec") => "#",
-                _ => "",
-            };
 
             // This loop only adds new items to values of the hash map, so the order in which we
             // iterate over the values is not important.
@@ -1030,10 +1044,13 @@ impl CrateInfo {
                 .for_each(|(_, linked_symbols)| {
                     let mut symbols = missing_weak_lang_items
                         .iter()
-                        .map(|item| {
+                        .map(|(item, export_kind)| {
                             (
-                                format!("{prefix}{}", mangle_internal_symbol(tcx, item.as_str())),
-                                SymbolExportKind::Text,
+                                add_prefix(
+                                    mangle_internal_symbol(tcx, item.as_str()),
+                                    *export_kind,
+                                ),
+                                *export_kind,
                             )
                         })
                         .collect::<Vec<_>>();
@@ -1048,12 +1065,12 @@ impl CrateInfo {
                         // errors.
                         linked_symbols.extend(ALLOCATOR_METHODS.iter().map(|method| {
                             (
-                                format!(
-                                    "{prefix}{}",
+                                add_prefix(
                                     mangle_internal_symbol(
                                         tcx,
-                                        global_fn_name(method.name).as_str()
-                                    )
+                                        global_fn_name(method.name).as_str(),
+                                    ),
+                                    SymbolExportKind::Text,
                                 ),
                                 SymbolExportKind::Text,
                             )
diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
index f769b393528..7bd27eb3ef1 100644
--- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
+++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
@@ -95,17 +95,15 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
         // In these cases, we bail from performing further checks that are only meaningful for
         // functions (such as calling `fn_sig`, which ICEs if given a non-function). We also
         // report a delayed bug, just in case `check_attr` isn't doing its job.
-        let fn_sig = || {
+        let fn_sig = |attr_span| {
             use DefKind::*;
 
             let def_kind = tcx.def_kind(did);
             if let Fn | AssocFn | Variant | Ctor(..) = def_kind {
                 Some(tcx.fn_sig(did))
             } else {
-                tcx.dcx().span_delayed_bug(
-                    attr.span(),
-                    "this attribute can only be applied to functions",
-                );
+                tcx.dcx()
+                    .span_delayed_bug(attr_span, "this attribute can only be applied to functions");
                 None
             }
         };
@@ -142,6 +140,29 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
                         });
                     }
                 }
+                AttributeKind::TrackCaller(attr_span) => {
+                    let is_closure = tcx.is_closure_like(did.to_def_id());
+
+                    if !is_closure
+                        && let Some(fn_sig) = fn_sig(*attr_span)
+                        && fn_sig.skip_binder().abi() != ExternAbi::Rust
+                    {
+                        tcx.dcx().emit_err(errors::RequiresRustAbi { span: *attr_span });
+                    }
+                    if is_closure
+                        && !tcx.features().closure_track_caller()
+                        && !attr_span.allows_unstable(sym::closure_track_caller)
+                    {
+                        feature_err(
+                            &tcx.sess,
+                            sym::closure_track_caller,
+                            *attr_span,
+                            "`#[track_caller]` on closures is currently unstable",
+                        )
+                        .emit();
+                    }
+                    codegen_fn_attrs.flags |= CodegenFnAttrFlags::TRACK_CALLER
+                }
                 _ => {}
             }
         }
@@ -202,29 +223,6 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
                 }
             }
             sym::thread_local => codegen_fn_attrs.flags |= CodegenFnAttrFlags::THREAD_LOCAL,
-            sym::track_caller => {
-                let is_closure = tcx.is_closure_like(did.to_def_id());
-
-                if !is_closure
-                    && let Some(fn_sig) = fn_sig()
-                    && fn_sig.skip_binder().abi() != ExternAbi::Rust
-                {
-                    tcx.dcx().emit_err(errors::RequiresRustAbi { span: attr.span() });
-                }
-                if is_closure
-                    && !tcx.features().closure_track_caller()
-                    && !attr.span().allows_unstable(sym::closure_track_caller)
-                {
-                    feature_err(
-                        &tcx.sess,
-                        sym::closure_track_caller,
-                        attr.span(),
-                        "`#[track_caller]` on closures is currently unstable",
-                    )
-                    .emit();
-                }
-                codegen_fn_attrs.flags |= CodegenFnAttrFlags::TRACK_CALLER
-            }
             sym::export_name => {
                 if let Some(s) = attr.value_str() {
                     if s.as_str().contains('\0') {
diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs
index 523c9f2ad1c..23ed387a3ff 100644
--- a/compiler/rustc_codegen_ssa/src/lib.rs
+++ b/compiler/rustc_codegen_ssa/src/lib.rs
@@ -218,7 +218,7 @@ pub struct CrateInfo {
     pub target_cpu: String,
     pub target_features: Vec<String>,
     pub crate_types: Vec<CrateType>,
-    pub exported_symbols: UnordMap<CrateType, Vec<String>>,
+    pub exported_symbols: UnordMap<CrateType, Vec<(String, SymbolExportKind)>>,
     pub linked_symbols: FxIndexMap<CrateType, Vec<(String, SymbolExportKind)>>,
     pub local_crate_name: Symbol,
     pub compiler_builtins: Option<CrateNum>,
diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
index e1d8b7546cf..db5ac6a514f 100644
--- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
@@ -1123,7 +1123,7 @@ pub(super) fn transmute_immediate<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     // While optimizations will remove no-op transmutes, they might still be
     // there in debug or things that aren't no-op in MIR because they change
     // the Rust type but not the underlying layout/niche.
-    if from_scalar == to_scalar && from_backend_ty == to_backend_ty {
+    if from_scalar == to_scalar {
         return imm;
     }
 
@@ -1142,7 +1142,13 @@ pub(super) fn transmute_immediate<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     assume_scalar_range(bx, imm, from_scalar, from_backend_ty);
 
     imm = match (from_scalar.primitive(), to_scalar.primitive()) {
-        (Int(..) | Float(_), Int(..) | Float(_)) => bx.bitcast(imm, to_backend_ty),
+        (Int(..) | Float(_), Int(..) | Float(_)) => {
+            if from_backend_ty == to_backend_ty {
+                imm
+            } else {
+                bx.bitcast(imm, to_backend_ty)
+            }
+        }
         (Pointer(..), Pointer(..)) => bx.pointercast(imm, to_backend_ty),
         (Int(..), Pointer(..)) => bx.ptradd(bx.const_null(bx.type_ptr()), imm),
         (Pointer(..), Int(..)) => {
diff --git a/compiler/rustc_const_eval/src/check_consts/ops.rs b/compiler/rustc_const_eval/src/check_consts/ops.rs
index 887275e7294..f5b7a6066c8 100644
--- a/compiler/rustc_const_eval/src/check_consts/ops.rs
+++ b/compiler/rustc_const_eval/src/check_consts/ops.rs
@@ -600,11 +600,13 @@ impl<'tcx> NonConstOp<'tcx> for EscapingMutBorrow {
                 kind: ccx.const_kind(),
                 teach: ccx.tcx.sess.teach(E0764),
             }),
-            hir::BorrowKind::Ref => ccx.dcx().create_err(errors::MutableRefEscaping {
-                span,
-                kind: ccx.const_kind(),
-                teach: ccx.tcx.sess.teach(E0764),
-            }),
+            hir::BorrowKind::Ref | hir::BorrowKind::Pin => {
+                ccx.dcx().create_err(errors::MutableRefEscaping {
+                    span,
+                    kind: ccx.const_kind(),
+                    teach: ccx.tcx.sess.teach(E0764),
+                })
+            }
         }
     }
 }
diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs
index daeca43169d..4855fc58d03 100644
--- a/compiler/rustc_driver_impl/src/lib.rs
+++ b/compiler/rustc_driver_impl/src/lib.rs
@@ -53,13 +53,13 @@ use rustc_metadata::locator;
 use rustc_middle::ty::TyCtxt;
 use rustc_parse::{new_parser_from_file, new_parser_from_source_str, unwrap_or_emit_fatal};
 use rustc_session::config::{
-    CG_OPTIONS, CrateType, ErrorOutputType, Input, OptionDesc, OutFileName, OutputType,
+    CG_OPTIONS, CrateType, ErrorOutputType, Input, OptionDesc, OutFileName, OutputType, Sysroot,
     UnstableOptions, Z_OPTIONS, nightly_options, parse_target_triple,
 };
 use rustc_session::getopts::{self, Matches};
 use rustc_session::lint::{Lint, LintId};
 use rustc_session::output::{CRATE_TYPES, collect_crate_types, invalid_output_for_target};
-use rustc_session::{EarlyDiagCtxt, Session, config, filesearch};
+use rustc_session::{EarlyDiagCtxt, Session, config};
 use rustc_span::FileName;
 use rustc_span::def_id::LOCAL_CRATE;
 use rustc_target::json::ToJson;
@@ -662,7 +662,7 @@ fn print_crate_info(
                 println_info!("{}", targets.join("\n"));
             }
             HostTuple => println_info!("{}", rustc_session::config::host_tuple()),
-            Sysroot => println_info!("{}", sess.sysroot.display()),
+            Sysroot => println_info!("{}", sess.opts.sysroot.path().display()),
             TargetLibdir => println_info!("{}", sess.target_tlib_path.dir.display()),
             TargetSpecJson => {
                 println_info!("{}", serde_json::to_string_pretty(&sess.target.to_json()).unwrap());
@@ -1114,8 +1114,8 @@ fn get_backend_from_raw_matches(
     let debug_flags = matches.opt_strs("Z");
     let backend_name = debug_flags.iter().find_map(|x| x.strip_prefix("codegen-backend="));
     let target = parse_target_triple(early_dcx, matches);
-    let sysroot = filesearch::materialize_sysroot(matches.opt_str("sysroot").map(PathBuf::from));
-    let target = config::build_target_config(early_dcx, &target, &sysroot);
+    let sysroot = Sysroot::new(matches.opt_str("sysroot").map(PathBuf::from));
+    let target = config::build_target_config(early_dcx, &target, sysroot.path());
 
     get_codegen_backend(early_dcx, &sysroot, backend_name, &target)
 }
diff --git a/compiler/rustc_error_messages/Cargo.toml b/compiler/rustc_error_messages/Cargo.toml
index 5dc582b9c3a..0951859fa53 100644
--- a/compiler/rustc_error_messages/Cargo.toml
+++ b/compiler/rustc_error_messages/Cargo.toml
@@ -16,7 +16,6 @@ rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_span = { path = "../rustc_span" }
-smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 tracing = "0.1"
 unic-langid = { version = "0.9.0", features = ["macros"] }
 # tidy-alphabetical-end
diff --git a/compiler/rustc_error_messages/src/lib.rs b/compiler/rustc_error_messages/src/lib.rs
index 194fc2450ba..4e4345cfe0f 100644
--- a/compiler/rustc_error_messages/src/lib.rs
+++ b/compiler/rustc_error_messages/src/lib.rs
@@ -8,7 +8,7 @@
 
 use std::borrow::Cow;
 use std::error::Error;
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use std::sync::{Arc, LazyLock};
 use std::{fmt, fs, io};
 
@@ -21,7 +21,6 @@ use intl_memoizer::concurrent::IntlLangMemoizer;
 use rustc_data_structures::sync::{DynSend, IntoDynSyncSend};
 use rustc_macros::{Decodable, Encodable};
 use rustc_span::Span;
-use smallvec::SmallVec;
 use tracing::{instrument, trace};
 pub use unic_langid::{LanguageIdentifier, langid};
 
@@ -107,7 +106,7 @@ impl From<Vec<FluentError>> for TranslationBundleError {
 /// (overriding any conflicting messages).
 #[instrument(level = "trace")]
 pub fn fluent_bundle(
-    sysroot_candidates: SmallVec<[PathBuf; 2]>,
+    sysroot_candidates: &[&Path],
     requested_locale: Option<LanguageIdentifier>,
     additional_ftl_path: Option<&Path>,
     with_directionality_markers: bool,
@@ -141,7 +140,8 @@ pub fn fluent_bundle(
     // If the user requests the default locale then don't try to load anything.
     if let Some(requested_locale) = requested_locale {
         let mut found_resources = false;
-        for mut sysroot in sysroot_candidates {
+        for sysroot in sysroot_candidates {
+            let mut sysroot = sysroot.to_path_buf();
             sysroot.push("share");
             sysroot.push("locale");
             sysroot.push(requested_locale.to_string());
diff --git a/compiler/rustc_expand/messages.ftl b/compiler/rustc_expand/messages.ftl
index 8b7c47dad99..b7555bba28e 100644
--- a/compiler/rustc_expand/messages.ftl
+++ b/compiler/rustc_expand/messages.ftl
@@ -62,7 +62,7 @@ expand_feature_not_allowed =
 expand_feature_removed =
     feature has been removed
     .label = feature has been removed
-    .note = removed in {$removed_rustc_version} (you are using {$current_rustc_version}){$pull_note}
+    .note = removed in {$removed_rustc_version}{$pull_note}
     .reason = {$reason}
 
 expand_glob_delegation_outside_impls =
diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs
index 9a359e9b031..170ac39d1ec 100644
--- a/compiler/rustc_expand/src/config.rs
+++ b/compiler/rustc_expand/src/config.rs
@@ -92,7 +92,6 @@ pub fn features(sess: &Session, krate_attrs: &[Attribute], crate_name: Symbol) -
                     span: mi.span(),
                     reason: f.reason.map(|reason| FeatureRemovedReason { reason }),
                     removed_rustc_version: f.feature.since,
-                    current_rustc_version: sess.cfg_version,
                     pull_note,
                 });
                 continue;
diff --git a/compiler/rustc_expand/src/errors.rs b/compiler/rustc_expand/src/errors.rs
index 714ba3bf0f4..fe4d2af97a0 100644
--- a/compiler/rustc_expand/src/errors.rs
+++ b/compiler/rustc_expand/src/errors.rs
@@ -162,7 +162,6 @@ pub(crate) struct FeatureRemoved<'a> {
     #[subdiagnostic]
     pub reason: Option<FeatureRemovedReason<'a>>,
     pub removed_rustc_version: &'a str,
-    pub current_rustc_version: &'a str,
     pub pull_note: String,
 }
 
@@ -444,7 +443,7 @@ pub(crate) struct InvalidFragmentSpecifier {
     #[primary_span]
     pub span: Span,
     pub fragment: Ident,
-    pub help: String,
+    pub help: &'static str,
 }
 
 #[derive(Diagnostic)]
diff --git a/compiler/rustc_expand/src/mbe.rs b/compiler/rustc_expand/src/mbe.rs
index 4ff8c02bcdb..3082c881a7a 100644
--- a/compiler/rustc_expand/src/mbe.rs
+++ b/compiler/rustc_expand/src/mbe.rs
@@ -78,7 +78,13 @@ enum TokenTree {
     /// only covers the ident, e.g. `var`.)
     MetaVar(Span, Ident),
     /// e.g., `$var:expr`. Only appears on the LHS.
-    MetaVarDecl(Span, Ident /* name to bind */, Option<NonterminalKind>),
+    MetaVarDecl {
+        span: Span,
+        /// Name to bind.
+        name: Ident,
+        /// The fragment specifier.
+        kind: NonterminalKind,
+    },
     /// A meta-variable expression inside `${...}`.
     MetaVarExpr(DelimSpan, MetaVarExpr),
 }
@@ -102,7 +108,7 @@ impl TokenTree {
         match *self {
             TokenTree::Token(Token { span, .. })
             | TokenTree::MetaVar(span, _)
-            | TokenTree::MetaVarDecl(span, _, _) => span,
+            | TokenTree::MetaVarDecl { span, .. } => span,
             TokenTree::Delimited(span, ..)
             | TokenTree::MetaVarExpr(span, _)
             | TokenTree::Sequence(span, _) => span.entire(),
diff --git a/compiler/rustc_expand/src/mbe/diagnostics.rs b/compiler/rustc_expand/src/mbe/diagnostics.rs
index 698492f42e2..99aa376626d 100644
--- a/compiler/rustc_expand/src/mbe/diagnostics.rs
+++ b/compiler/rustc_expand/src/mbe/diagnostics.rs
@@ -24,6 +24,7 @@ pub(super) fn failed_to_match_macro(
     arg: TokenStream,
     lhses: &[Vec<MatcherLoc>],
 ) -> (Span, ErrorGuaranteed) {
+    debug!("failed to match macro");
     // An error occurred, try the expansion again, tracking the expansion closely for better
     // diagnostics.
     let mut tracker = CollectTrackerAndEmitter::new(psess.dcx(), sp);
diff --git a/compiler/rustc_expand/src/mbe/macro_check.rs b/compiler/rustc_expand/src/mbe/macro_check.rs
index 3cd803c3e84..dc2d46c4a14 100644
--- a/compiler/rustc_expand/src/mbe/macro_check.rs
+++ b/compiler/rustc_expand/src/mbe/macro_check.rs
@@ -117,7 +117,6 @@ use rustc_session::parse::ParseSess;
 use rustc_span::{ErrorGuaranteed, MacroRulesNormalizedIdent, Span, kw};
 use smallvec::SmallVec;
 
-use super::quoted::VALID_FRAGMENT_NAMES_MSG;
 use crate::errors;
 use crate::mbe::{KleeneToken, TokenTree};
 
@@ -263,14 +262,7 @@ fn check_binders(
             }
         }
         // Similarly, this can only happen when checking a toplevel macro.
-        TokenTree::MetaVarDecl(span, name, kind) => {
-            if kind.is_none() && node_id != DUMMY_NODE_ID {
-                psess.dcx().emit_err(errors::MissingFragmentSpecifier {
-                    span,
-                    add_span: span.shrink_to_hi(),
-                    valid: VALID_FRAGMENT_NAMES_MSG,
-                });
-            }
+        TokenTree::MetaVarDecl { span, name, .. } => {
             if !macros.is_empty() {
                 psess.dcx().span_bug(span, "unexpected MetaVarDecl in nested lhs");
             }
@@ -339,7 +331,7 @@ fn check_occurrences(
 ) {
     match *rhs {
         TokenTree::Token(..) => {}
-        TokenTree::MetaVarDecl(span, _name, _kind) => {
+        TokenTree::MetaVarDecl { span, .. } => {
             psess.dcx().span_bug(span, "unexpected MetaVarDecl in rhs")
         }
         TokenTree::MetaVar(span, name) => {
diff --git a/compiler/rustc_expand/src/mbe/macro_parser.rs b/compiler/rustc_expand/src/mbe/macro_parser.rs
index c78beb40688..802e43209a5 100644
--- a/compiler/rustc_expand/src/mbe/macro_parser.rs
+++ b/compiler/rustc_expand/src/mbe/macro_parser.rs
@@ -122,7 +122,7 @@ pub(crate) enum MatcherLoc {
     MetaVarDecl {
         span: Span,
         bind: Ident,
-        kind: Option<NonterminalKind>,
+        kind: NonterminalKind,
         next_metavar: usize,
         seq_depth: usize,
     },
@@ -151,12 +151,7 @@ impl Display for MatcherLoc {
                 write!(f, "{}", token_descr(token))
             }
             MatcherLoc::MetaVarDecl { bind, kind, .. } => {
-                write!(f, "meta-variable `${bind}")?;
-                if let Some(kind) = kind {
-                    write!(f, ":{kind}")?;
-                }
-                write!(f, "`")?;
-                Ok(())
+                write!(f, "meta-variable `${bind}:{kind}`")
             }
             MatcherLoc::Eof => f.write_str("end of macro"),
 
@@ -220,7 +215,7 @@ pub(super) fn compute_locs(matcher: &[TokenTree]) -> Vec<MatcherLoc> {
                         seq_depth,
                     };
                 }
-                &TokenTree::MetaVarDecl(span, bind, kind) => {
+                &TokenTree::MetaVarDecl { span, name: bind, kind } => {
                     locs.push(MatcherLoc::MetaVarDecl {
                         span,
                         bind,
@@ -330,7 +325,7 @@ pub(super) fn count_metavar_decls(matcher: &[TokenTree]) -> usize {
     matcher
         .iter()
         .map(|tt| match tt {
-            TokenTree::MetaVarDecl(..) => 1,
+            TokenTree::MetaVarDecl { .. } => 1,
             TokenTree::Sequence(_, seq) => seq.num_captures,
             TokenTree::Delimited(.., delim) => count_metavar_decls(&delim.tts),
             TokenTree::Token(..) => 0,
@@ -551,18 +546,12 @@ impl TtParser {
                     mp.idx = idx_first;
                     self.cur_mps.push(mp);
                 }
-                &MatcherLoc::MetaVarDecl { span, kind, .. } => {
+                &MatcherLoc::MetaVarDecl { kind, .. } => {
                     // Built-in nonterminals never start with these tokens, so we can eliminate
                     // them from consideration. We use the span of the metavariable declaration
                     // to determine any edition-specific matching behavior for non-terminals.
-                    if let Some(kind) = kind {
-                        if Parser::nonterminal_may_begin_with(kind, token) {
-                            self.bb_mps.push(mp);
-                        }
-                    } else {
-                        // E.g. `$e` instead of `$e:expr`, reported as a hard error if actually used.
-                        // Both this check and the one in `nameize` are necessary, surprisingly.
-                        return Some(Error(span, "missing fragment specifier".to_string()));
+                    if Parser::nonterminal_may_begin_with(kind, token) {
+                        self.bb_mps.push(mp);
                     }
                 }
                 MatcherLoc::Eof => {
@@ -666,11 +655,7 @@ impl TtParser {
                     let mut mp = self.bb_mps.pop().unwrap();
                     let loc = &matcher[mp.idx];
                     if let &MatcherLoc::MetaVarDecl {
-                        span,
-                        kind: Some(kind),
-                        next_metavar,
-                        seq_depth,
-                        ..
+                        span, kind, next_metavar, seq_depth, ..
                     } = loc
                     {
                         // We use the span of the metavariable declaration to determine any
@@ -715,7 +700,7 @@ impl TtParser {
             .bb_mps
             .iter()
             .map(|mp| match &matcher[mp.idx] {
-                MatcherLoc::MetaVarDecl { bind, kind: Some(kind), .. } => {
+                MatcherLoc::MetaVarDecl { bind, kind, .. } => {
                     format!("{kind} ('{bind}')")
                 }
                 _ => unreachable!(),
@@ -745,19 +730,13 @@ impl TtParser {
         // `NamedParseResult`. Otherwise, it's an error.
         let mut ret_val = FxHashMap::default();
         for loc in matcher {
-            if let &MatcherLoc::MetaVarDecl { span, bind, kind, .. } = loc {
-                if kind.is_some() {
-                    match ret_val.entry(MacroRulesNormalizedIdent::new(bind)) {
-                        Vacant(spot) => spot.insert(res.next().unwrap()),
-                        Occupied(..) => {
-                            return Error(span, format!("duplicated bind name: {bind}"));
-                        }
-                    };
-                } else {
-                    // E.g. `$e` instead of `$e:expr`, reported as a hard error if actually used.
-                    // Both this check and the one in `parse_tt_inner` are necessary, surprisingly.
-                    return Error(span, "missing fragment specifier".to_string());
-                }
+            if let &MatcherLoc::MetaVarDecl { span, bind, .. } = loc {
+                match ret_val.entry(MacroRulesNormalizedIdent::new(bind)) {
+                    Vacant(spot) => spot.insert(res.next().unwrap()),
+                    Occupied(..) => {
+                        return Error(span, format!("duplicated bind name: {bind}"));
+                    }
+                };
             }
         }
         Success(ret_val)
diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs
index 783f061ec6c..432ab324740 100644
--- a/compiler/rustc_expand/src/mbe/macro_rules.rs
+++ b/compiler/rustc_expand/src/mbe/macro_rules.rs
@@ -392,7 +392,7 @@ pub fn compile_declarative_macro(
 
     let lhs_nm = Ident::new(sym::lhs, span);
     let rhs_nm = Ident::new(sym::rhs, span);
-    let tt_spec = Some(NonterminalKind::TT);
+    let tt_spec = NonterminalKind::TT;
     let macro_rules = macro_def.macro_rules;
 
     // Parse the macro_rules! invocation
@@ -407,9 +407,9 @@ pub fn compile_declarative_macro(
             DelimSpan::dummy(),
             mbe::SequenceRepetition {
                 tts: vec![
-                    mbe::TokenTree::MetaVarDecl(span, lhs_nm, tt_spec),
+                    mbe::TokenTree::MetaVarDecl { span, name: lhs_nm, kind: tt_spec },
                     mbe::TokenTree::token(token::FatArrow, span),
-                    mbe::TokenTree::MetaVarDecl(span, rhs_nm, tt_spec),
+                    mbe::TokenTree::MetaVarDecl { span, name: rhs_nm, kind: tt_spec },
                 ],
                 separator: Some(Token::new(
                     if macro_rules { token::Semi } else { token::Comma },
@@ -448,6 +448,7 @@ pub fn compile_declarative_macro(
         match tt_parser.parse_tt(&mut Cow::Owned(parser), &argument_gram, &mut NoopTracker) {
             Success(m) => m,
             Failure(()) => {
+                debug!("failed to parse macro tt");
                 // The fast `NoopTracker` doesn't have any info on failure, so we need to retry it
                 // with another one that gives us the information we need.
                 // For this we need to reclone the macro body as the previous parser consumed it.
@@ -616,7 +617,7 @@ fn is_empty_token_tree(sess: &Session, seq: &mbe::SequenceRepetition) -> bool {
         let mut iter = seq.tts.iter().peekable();
         while let Some(tt) = iter.next() {
             match tt {
-                mbe::TokenTree::MetaVarDecl(_, _, Some(NonterminalKind::Vis)) => {}
+                mbe::TokenTree::MetaVarDecl { kind: NonterminalKind::Vis, .. } => {}
                 mbe::TokenTree::Token(t @ Token { kind: DocComment(..), .. }) => {
                     let mut now = t;
                     while let Some(&mbe::TokenTree::Token(
@@ -651,7 +652,7 @@ fn check_redundant_vis_repetition(
 ) {
     let is_zero_or_one: bool = seq.kleene.op == KleeneOp::ZeroOrOne;
     let is_vis = seq.tts.first().map_or(false, |tt| {
-        matches!(tt, mbe::TokenTree::MetaVarDecl(_, _, Some(NonterminalKind::Vis)))
+        matches!(tt, mbe::TokenTree::MetaVarDecl { kind: NonterminalKind::Vis, .. })
     });
 
     if is_vis && is_zero_or_one {
@@ -678,7 +679,7 @@ fn check_lhs_no_empty_seq(sess: &Session, tts: &[mbe::TokenTree]) -> Result<(),
         match tt {
             TokenTree::Token(..)
             | TokenTree::MetaVar(..)
-            | TokenTree::MetaVarDecl(..)
+            | TokenTree::MetaVarDecl { .. }
             | TokenTree::MetaVarExpr(..) => (),
             TokenTree::Delimited(.., del) => check_lhs_no_empty_seq(sess, &del.tts)?,
             TokenTree::Sequence(span, seq) => {
@@ -777,7 +778,7 @@ impl<'tt> FirstSets<'tt> {
                 match tt {
                     TokenTree::Token(..)
                     | TokenTree::MetaVar(..)
-                    | TokenTree::MetaVarDecl(..)
+                    | TokenTree::MetaVarDecl { .. }
                     | TokenTree::MetaVarExpr(..) => {
                         first.replace_with(TtHandle::TtRef(tt));
                     }
@@ -845,7 +846,7 @@ impl<'tt> FirstSets<'tt> {
             match tt {
                 TokenTree::Token(..)
                 | TokenTree::MetaVar(..)
-                | TokenTree::MetaVarDecl(..)
+                | TokenTree::MetaVarDecl { .. }
                 | TokenTree::MetaVarExpr(..) => {
                     first.add_one(TtHandle::TtRef(tt));
                     return first;
@@ -1084,7 +1085,7 @@ fn check_matcher_core<'tt>(
         match token {
             TokenTree::Token(..)
             | TokenTree::MetaVar(..)
-            | TokenTree::MetaVarDecl(..)
+            | TokenTree::MetaVarDecl { .. }
             | TokenTree::MetaVarExpr(..) => {
                 if token_can_be_followed_by_any(token) {
                     // don't need to track tokens that work with any,
@@ -1152,7 +1153,7 @@ fn check_matcher_core<'tt>(
         // Now `last` holds the complete set of NT tokens that could
         // end the sequence before SUFFIX. Check that every one works with `suffix`.
         for tt in &last.tokens {
-            if let &TokenTree::MetaVarDecl(span, name, Some(kind)) = tt.get() {
+            if let &TokenTree::MetaVarDecl { span, name, kind } = tt.get() {
                 for next_token in &suffix_first.tokens {
                     let next_token = next_token.get();
 
@@ -1172,11 +1173,11 @@ fn check_matcher_core<'tt>(
                         )
                     {
                         // It is suggestion to use pat_param, for example: $x:pat -> $x:pat_param.
-                        let suggestion = quoted_tt_to_string(&TokenTree::MetaVarDecl(
+                        let suggestion = quoted_tt_to_string(&TokenTree::MetaVarDecl {
                             span,
                             name,
-                            Some(NonterminalKind::Pat(PatParam { inferred: false })),
-                        ));
+                            kind: NonterminalKind::Pat(PatParam { inferred: false }),
+                        });
                         sess.psess.buffer_lint(
                             RUST_2021_INCOMPATIBLE_OR_PATTERNS,
                             span,
@@ -1212,11 +1213,11 @@ fn check_matcher_core<'tt>(
                                 && sess.psess.edition.at_least_rust_2021()
                                 && next_token.is_token(&token::Or)
                             {
-                                let suggestion = quoted_tt_to_string(&TokenTree::MetaVarDecl(
+                                let suggestion = quoted_tt_to_string(&TokenTree::MetaVarDecl {
                                     span,
                                     name,
-                                    Some(NonterminalKind::Pat(PatParam { inferred: false })),
-                                ));
+                                    kind: NonterminalKind::Pat(PatParam { inferred: false }),
+                                });
                                 err.span_suggestion(
                                     span,
                                     "try a `pat_param` fragment specifier instead",
@@ -1254,7 +1255,7 @@ fn check_matcher_core<'tt>(
 }
 
 fn token_can_be_followed_by_any(tok: &mbe::TokenTree) -> bool {
-    if let mbe::TokenTree::MetaVarDecl(_, _, Some(kind)) = *tok {
+    if let mbe::TokenTree::MetaVarDecl { kind, .. } = *tok {
         frag_can_be_followed_by_any(kind)
     } else {
         // (Non NT's can always be followed by anything in matchers.)
@@ -1367,7 +1368,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
                         }
                         _ => IsInFollow::No(TOKENS),
                     },
-                    TokenTree::MetaVarDecl(_, _, Some(NonterminalKind::Block)) => IsInFollow::Yes,
+                    TokenTree::MetaVarDecl { kind: NonterminalKind::Block, .. } => IsInFollow::Yes,
                     _ => IsInFollow::No(TOKENS),
                 }
             }
@@ -1400,11 +1401,10 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
                             }
                         }
                     },
-                    TokenTree::MetaVarDecl(
-                        _,
-                        _,
-                        Some(NonterminalKind::Ident | NonterminalKind::Ty | NonterminalKind::Path),
-                    ) => IsInFollow::Yes,
+                    TokenTree::MetaVarDecl {
+                        kind: NonterminalKind::Ident | NonterminalKind::Ty | NonterminalKind::Path,
+                        ..
+                    } => IsInFollow::Yes,
                     _ => IsInFollow::No(TOKENS),
                 }
             }
@@ -1416,8 +1416,7 @@ fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String {
     match tt {
         mbe::TokenTree::Token(token) => pprust::token_to_string(token).into(),
         mbe::TokenTree::MetaVar(_, name) => format!("${name}"),
-        mbe::TokenTree::MetaVarDecl(_, name, Some(kind)) => format!("${name}:{kind}"),
-        mbe::TokenTree::MetaVarDecl(_, name, None) => format!("${name}:"),
+        mbe::TokenTree::MetaVarDecl { name, kind, .. } => format!("${name}:{kind}"),
         _ => panic!(
             "{}",
             "unexpected mbe::TokenTree::{Sequence or Delimited} \
diff --git a/compiler/rustc_expand/src/mbe/quoted.rs b/compiler/rustc_expand/src/mbe/quoted.rs
index 0c2362f23bc..2daa4e71558 100644
--- a/compiler/rustc_expand/src/mbe/quoted.rs
+++ b/compiler/rustc_expand/src/mbe/quoted.rs
@@ -54,66 +54,78 @@ pub(super) fn parse(
         // Given the parsed tree, if there is a metavar and we are expecting matchers, actually
         // parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`).
         let tree = parse_tree(tree, &mut iter, parsing_patterns, sess, node_id, features, edition);
-        match tree {
-            TokenTree::MetaVar(start_sp, ident) if parsing_patterns => {
-                // Not consuming the next token immediately, as it may not be a colon
-                let span = match iter.peek() {
-                    Some(&tokenstream::TokenTree::Token(
-                        Token { kind: token::Colon, span: colon_span },
-                        _,
-                    )) => {
-                        // Consume the colon first
-                        iter.next();
-
-                        // It's ok to consume the next tree no matter how,
-                        // since if it's not a token then it will be an invalid declaration.
-                        match iter.next() {
-                            Some(tokenstream::TokenTree::Token(token, _)) => match token.ident() {
-                                Some((fragment, _)) => {
-                                    let span = token.span.with_lo(start_sp.lo());
-                                    let edition = || {
-                                        // FIXME(#85708) - once we properly decode a foreign
-                                        // crate's `SyntaxContext::root`, then we can replace
-                                        // this with just `span.edition()`. A
-                                        // `SyntaxContext::root()` from the current crate will
-                                        // have the edition of the current crate, and a
-                                        // `SyntaxContext::root()` from a foreign crate will
-                                        // have the edition of that crate (which we manually
-                                        // retrieve via the `edition` parameter).
-                                        if !span.from_expansion() {
-                                            edition
-                                        } else {
-                                            span.edition()
-                                        }
-                                    };
-                                    let kind = NonterminalKind::from_symbol(fragment.name, edition)
-                                        .unwrap_or_else(|| {
-                                            sess.dcx().emit_err(errors::InvalidFragmentSpecifier {
-                                                span,
-                                                fragment,
-                                                help: VALID_FRAGMENT_NAMES_MSG.into(),
-                                            });
-                                            NonterminalKind::Ident
-                                        });
-                                    result.push(TokenTree::MetaVarDecl(span, ident, Some(kind)));
-                                    continue;
-                                }
-                                _ => token.span,
-                            },
-                            // Invalid, return a nice source location
-                            _ => colon_span.with_lo(start_sp.lo()),
-                        }
-                    }
-                    // Whether it's none or some other tree, it doesn't belong to
-                    // the current meta variable, returning the original span.
-                    _ => start_sp,
-                };
 
-                result.push(TokenTree::MetaVarDecl(span, ident, None));
-            }
+        if !parsing_patterns {
+            // No matchers allowed, nothing to process here
+            result.push(tree);
+            continue;
+        }
+
+        let TokenTree::MetaVar(start_sp, ident) = tree else {
+            // Not a metavariable, just return the tree
+            result.push(tree);
+            continue;
+        };
 
-            // Not a metavar or no matchers allowed, so just return the tree
-            _ => result.push(tree),
+        // Push a metavariable with no fragment specifier at the given span
+        let mut missing_fragment_specifier = |span| {
+            sess.dcx().emit_err(errors::MissingFragmentSpecifier {
+                span,
+                add_span: span.shrink_to_hi(),
+                valid: VALID_FRAGMENT_NAMES_MSG,
+            });
+
+            // Fall back to a `TokenTree` since that will match anything if we continue expanding.
+            result.push(TokenTree::MetaVarDecl { span, name: ident, kind: NonterminalKind::TT });
+        };
+
+        // Not consuming the next token immediately, as it may not be a colon
+        if let Some(peek) = iter.peek()
+            && let tokenstream::TokenTree::Token(token, _spacing) = peek
+            && let Token { kind: token::Colon, span: colon_span } = token
+        {
+            // Next token is a colon; consume it
+            iter.next();
+
+            // It's ok to consume the next tree no matter how,
+            // since if it's not a token then it will be an invalid declaration.
+            let Some(tokenstream::TokenTree::Token(token, _)) = iter.next() else {
+                // Invalid, return a nice source location as `var:`
+                missing_fragment_specifier(colon_span.with_lo(start_sp.lo()));
+                continue;
+            };
+
+            let Some((fragment, _)) = token.ident() else {
+                // No identifier for the fragment specifier;
+                missing_fragment_specifier(token.span);
+                continue;
+            };
+
+            let span = token.span.with_lo(start_sp.lo());
+            let edition = || {
+                // FIXME(#85708) - once we properly decode a foreign
+                // crate's `SyntaxContext::root`, then we can replace
+                // this with just `span.edition()`. A
+                // `SyntaxContext::root()` from the current crate will
+                // have the edition of the current crate, and a
+                // `SyntaxContext::root()` from a foreign crate will
+                // have the edition of that crate (which we manually
+                // retrieve via the `edition` parameter).
+                if !span.from_expansion() { edition } else { span.edition() }
+            };
+            let kind = NonterminalKind::from_symbol(fragment.name, edition).unwrap_or_else(|| {
+                sess.dcx().emit_err(errors::InvalidFragmentSpecifier {
+                    span,
+                    fragment,
+                    help: VALID_FRAGMENT_NAMES_MSG,
+                });
+                NonterminalKind::TT
+            });
+            result.push(TokenTree::MetaVarDecl { span, name: ident, kind });
+        } else {
+            // Whether it's none or some other tree, it doesn't belong to
+            // the current meta variable, returning the original span.
+            missing_fragment_specifier(start_sp);
         }
     }
     result
diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs
index 0520be5fbae..a8c4a9e4b1b 100644
--- a/compiler/rustc_expand/src/mbe/transcribe.rs
+++ b/compiler/rustc_expand/src/mbe/transcribe.rs
@@ -283,7 +283,7 @@ pub(super) fn transcribe<'a>(
             }
 
             // There should be no meta-var declarations in the invocation of a macro.
-            mbe::TokenTree::MetaVarDecl(..) => panic!("unexpected `TokenTree::MetaVarDecl`"),
+            mbe::TokenTree::MetaVarDecl { .. } => panic!("unexpected `TokenTree::MetaVarDecl`"),
         }
     }
 }
@@ -776,7 +776,7 @@ fn lockstep_iter_size(
                 size.with(lockstep_iter_size(tt, interpolations, repeats))
             })
         }
-        TokenTree::MetaVar(_, name) | TokenTree::MetaVarDecl(_, name, _) => {
+        TokenTree::MetaVar(_, name) | TokenTree::MetaVarDecl { name, .. } => {
             let name = MacroRulesNormalizedIdent::new(*name);
             match lookup_cur_matched(name, interpolations, repeats) {
                 Some(matched) => match matched {
diff --git a/compiler/rustc_expand/src/stats.rs b/compiler/rustc_expand/src/stats.rs
index 6b2ad30dffd..b4c4eac028f 100644
--- a/compiler/rustc_expand/src/stats.rs
+++ b/compiler/rustc_expand/src/stats.rs
@@ -15,15 +15,11 @@ pub struct MacroStat {
     /// Number of uses of the macro.
     pub uses: usize,
 
-    /// Net increase in number of lines of code (when pretty-printed), i.e.
-    /// `lines(output) - lines(invocation)`. Can be negative because a macro
-    /// output may be smaller than the invocation.
-    pub lines: isize,
-
-    /// Net increase in number of lines of code (when pretty-printed), i.e.
-    /// `bytes(output) - bytes(invocation)`. Can be negative because a macro
-    /// output may be smaller than the invocation.
-    pub bytes: isize,
+    /// Number of lines of code (when pretty-printed).
+    pub lines: usize,
+
+    /// Number of bytes of code (when pretty-printed).
+    pub bytes: usize,
 }
 
 pub(crate) fn elems_to_string<T>(elems: &SmallVec<[T; 1]>, f: impl Fn(&T) -> String) -> String {
@@ -131,16 +127,12 @@ pub(crate) fn update_macro_stats(
     input: &str,
     fragment: &AstFragment,
 ) {
-    fn lines_and_bytes(s: &str) -> (usize, usize) {
-        (s.trim_end().split('\n').count(), s.len())
-    }
-
     // Measure the size of the output by pretty-printing it and counting
     // the lines and bytes.
     let name = Symbol::intern(&pprust::path_to_string(path));
     let output = fragment.to_string();
-    let (in_l, in_b) = lines_and_bytes(input);
-    let (out_l, out_b) = lines_and_bytes(&output);
+    let num_lines = output.trim_end().split('\n').count();
+    let num_bytes = output.len();
 
     // This code is useful for debugging `-Zmacro-stats`. For every
     // invocation it prints the full input and output.
@@ -157,7 +149,7 @@ pub(crate) fn update_macro_stats(
             {name}: [{crate_name}] ({fragment_kind:?}) {span}\n\
             -------------------------------\n\
             {input}\n\
-            -- ({in_l} lines, {in_b} bytes) --> ({out_l} lines, {out_b} bytes) --\n\
+            -- {num_lines} lines, {num_bytes} bytes --\n\
             {output}\n\
         "
         );
@@ -166,6 +158,6 @@ pub(crate) fn update_macro_stats(
     // The recorded size is the difference between the input and the output.
     let entry = ecx.macro_stats.entry((name, macro_kind)).or_insert(MacroStat::default());
     entry.uses += 1;
-    entry.lines += out_l as isize - in_l as isize;
-    entry.bytes += out_b as isize - in_b as isize;
+    entry.lines += num_lines;
+    entry.bytes += num_bytes;
 }
diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs
index 3540c9b4bf0..8c0f279e343 100644
--- a/compiler/rustc_feature/src/builtin_attrs.rs
+++ b/compiler/rustc_feature/src/builtin_attrs.rs
@@ -657,6 +657,19 @@ pub static BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
         EncodeCrossCrate::Yes, min_generic_const_args, experimental!(type_const),
     ),
 
+    // The `#[loop_match]` and `#[const_continue]` attributes are part of the
+    // lang experiment for RFC 3720 tracked in:
+    //
+    // - https://github.com/rust-lang/rust/issues/132306
+    gated!(
+        const_continue, Normal, template!(Word), ErrorFollowing,
+        EncodeCrossCrate::No, loop_match, experimental!(const_continue)
+    ),
+    gated!(
+        loop_match, Normal, template!(Word), ErrorFollowing,
+        EncodeCrossCrate::No, loop_match, experimental!(loop_match)
+    ),
+
     // ==========================================================================
     // Internal attributes: Stability, deprecation, and unsafe:
     // ==========================================================================
diff --git a/compiler/rustc_feature/src/removed.rs b/compiler/rustc_feature/src/removed.rs
index 0cd090b25a4..c54b831e244 100644
--- a/compiler/rustc_feature/src/removed.rs
+++ b/compiler/rustc_feature/src/removed.rs
@@ -285,4 +285,18 @@ declare_features! (
     // -------------------------------------------------------------------------
     // feature-group-end: removed features
     // -------------------------------------------------------------------------
+
+
+    // -------------------------------------------------------------------------
+    // feature-group-start: removed library features
+    // -------------------------------------------------------------------------
+    //
+    // FIXME(#141617): we should have a better way to track removed library features, but we reuse
+    // the infrastructure here so users still get hints. The symbols used here can be remove from
+    // `symbol.rs` when that happens.
+    (removed, concat_idents, "CURRENT_RUSTC_VERSION", Some(29599),
+     Some("use the `${concat(..)}` metavariable expression instead"), 142704),
+    // -------------------------------------------------------------------------
+    // feature-group-end: removed library features
+    // -------------------------------------------------------------------------
 );
diff --git a/compiler/rustc_feature/src/unstable.rs b/compiler/rustc_feature/src/unstable.rs
index 91715851226..d9d5334615a 100644
--- a/compiler/rustc_feature/src/unstable.rs
+++ b/compiler/rustc_feature/src/unstable.rs
@@ -557,6 +557,8 @@ declare_features! (
     /// Allows using `#[link(kind = "link-arg", name = "...")]`
     /// to pass custom arguments to the linker.
     (unstable, link_arg_attribute, "1.76.0", Some(99427)),
+    /// Allows fused `loop`/`match` for direct intraprocedural jumps.
+    (incomplete, loop_match, "CURRENT_RUSTC_VERSION", Some(132306)),
     /// Give access to additional metadata about declarative macro meta-variables.
     (unstable, macro_metavar_expr, "1.61.0", Some(83527)),
     /// Provides a way to concatenate identifiers using metavariable expressions.
diff --git a/compiler/rustc_hir_analysis/messages.ftl b/compiler/rustc_hir_analysis/messages.ftl
index bd2252c1bf8..8c43d10b9e4 100644
--- a/compiler/rustc_hir_analysis/messages.ftl
+++ b/compiler/rustc_hir_analysis/messages.ftl
@@ -1,5 +1,5 @@
 hir_analysis_abi_custom_clothed_function =
-    items with the `"custom"` ABI can only be declared externally or defined via naked functions
+    items with the "custom" ABI can only be declared externally or defined via naked functions
     .suggestion = convert this to an `#[unsafe(naked)]` function
 
 hir_analysis_ambiguous_assoc_item = ambiguous associated {$assoc_kind} `{$assoc_ident}` in bounds of `{$qself}`
diff --git a/compiler/rustc_hir_analysis/src/check/entry.rs b/compiler/rustc_hir_analysis/src/check/entry.rs
index 3bad36da999..b556683e80a 100644
--- a/compiler/rustc_hir_analysis/src/check/entry.rs
+++ b/compiler/rustc_hir_analysis/src/check/entry.rs
@@ -1,14 +1,15 @@
 use std::ops::Not;
 
 use rustc_abi::ExternAbi;
+use rustc_attr_data_structures::{AttributeKind, find_attr};
 use rustc_hir as hir;
 use rustc_hir::Node;
 use rustc_infer::infer::TyCtxtInferExt;
 use rustc_middle::span_bug;
 use rustc_middle::ty::{self, TyCtxt, TypingMode};
 use rustc_session::config::EntryFnType;
+use rustc_span::Span;
 use rustc_span::def_id::{CRATE_DEF_ID, DefId, LocalDefId};
-use rustc_span::{Span, sym};
 use rustc_trait_selection::error_reporting::InferCtxtErrorExt;
 use rustc_trait_selection::traits::{self, ObligationCause, ObligationCauseCode};
 
@@ -98,8 +99,10 @@ fn check_main_fn_ty(tcx: TyCtxt<'_>, main_def_id: DefId) {
         error = true;
     }
 
-    for attr in tcx.get_attrs(main_def_id, sym::track_caller) {
-        tcx.dcx().emit_err(errors::TrackCallerOnMain { span: attr.span(), annotated: main_span });
+    if let Some(attr_span) =
+        find_attr!(tcx.get_all_attrs(main_def_id), AttributeKind::TrackCaller(span) => *span)
+    {
+        tcx.dcx().emit_err(errors::TrackCallerOnMain { span: attr_span, annotated: main_span });
         error = true;
     }
 
diff --git a/compiler/rustc_hir_pretty/src/lib.rs b/compiler/rustc_hir_pretty/src/lib.rs
index d3289e4cc6d..ff87d3dec6e 100644
--- a/compiler/rustc_hir_pretty/src/lib.rs
+++ b/compiler/rustc_hir_pretty/src/lib.rs
@@ -1335,6 +1335,10 @@ impl<'a> State<'a> {
                 self.word_nbsp("raw");
                 self.print_mutability(mutability, true);
             }
+            hir::BorrowKind::Pin => {
+                self.word_nbsp("pin");
+                self.print_mutability(mutability, true);
+            }
         }
         self.print_expr_cond_paren(expr, self.precedence(expr) < ExprPrecedence::Prefix);
     }
diff --git a/compiler/rustc_hir_typeck/messages.ftl b/compiler/rustc_hir_typeck/messages.ftl
index 258535f3742..c21b16c9f9f 100644
--- a/compiler/rustc_hir_typeck/messages.ftl
+++ b/compiler/rustc_hir_typeck/messages.ftl
@@ -79,6 +79,9 @@ hir_typeck_cast_unknown_pointer = cannot cast {$to ->
     .note = the type information given here is insufficient to check whether the pointer cast is valid
     .label_from = the type information given here is insufficient to check whether the pointer cast is valid
 
+hir_typeck_const_continue_bad_label =
+    `#[const_continue]` must break to a labeled block that participates in a `#[loop_match]`
+
 hir_typeck_const_select_must_be_const = this argument must be a `const fn`
     .help = consult the documentation on `const_eval_select` for more information
 
diff --git a/compiler/rustc_hir_typeck/src/errors.rs b/compiler/rustc_hir_typeck/src/errors.rs
index 5fea0c62843..3606c778fc4 100644
--- a/compiler/rustc_hir_typeck/src/errors.rs
+++ b/compiler/rustc_hir_typeck/src/errors.rs
@@ -1167,3 +1167,10 @@ pub(crate) struct AbiCannotBeCalled {
     pub span: Span,
     pub abi: ExternAbi,
 }
+
+#[derive(Diagnostic)]
+#[diag(hir_typeck_const_continue_bad_label)]
+pub(crate) struct ConstContinueBadLabel {
+    #[primary_span]
+    pub span: Span,
+}
diff --git a/compiler/rustc_hir_typeck/src/expr.rs b/compiler/rustc_hir_typeck/src/expr.rs
index 672f3bc67ce..2bc9dadb665 100644
--- a/compiler/rustc_hir_typeck/src/expr.rs
+++ b/compiler/rustc_hir_typeck/src/expr.rs
@@ -690,7 +690,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 self.check_named_place_expr(oprnd);
                 Ty::new_ptr(self.tcx, ty, mutbl)
             }
-            hir::BorrowKind::Ref => {
+            hir::BorrowKind::Ref | hir::BorrowKind::Pin => {
                 // Note: at this point, we cannot say what the best lifetime
                 // is to use for resulting pointer. We want to use the
                 // shortest lifetime possible so as to avoid spurious borrowck
@@ -706,7 +706,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 // whose address was taken can actually be made to live as long
                 // as it needs to live.
                 let region = self.next_region_var(infer::BorrowRegion(expr.span));
-                Ty::new_ref(self.tcx, region, ty, mutbl)
+                match kind {
+                    hir::BorrowKind::Ref => Ty::new_ref(self.tcx, region, ty, mutbl),
+                    hir::BorrowKind::Pin => Ty::new_pinned_ref(self.tcx, region, ty, mutbl),
+                    _ => unreachable!(),
+                }
             }
         }
     }
diff --git a/compiler/rustc_hir_typeck/src/loops.rs b/compiler/rustc_hir_typeck/src/loops.rs
index b06e0704b6f..80eab578f13 100644
--- a/compiler/rustc_hir_typeck/src/loops.rs
+++ b/compiler/rustc_hir_typeck/src/loops.rs
@@ -2,6 +2,8 @@ use std::collections::BTreeMap;
 use std::fmt;
 
 use Context::*;
+use rustc_ast::Label;
+use rustc_attr_data_structures::{AttributeKind, find_attr};
 use rustc_hir as hir;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::LocalDefId;
@@ -14,8 +16,9 @@ use rustc_span::hygiene::DesugaringKind;
 use rustc_span::{BytePos, Span};
 
 use crate::errors::{
-    BreakInsideClosure, BreakInsideCoroutine, BreakNonLoop, ContinueLabeledBlock, OutsideLoop,
-    OutsideLoopSuggestion, UnlabeledCfInWhileCondition, UnlabeledInLabeledBlock,
+    BreakInsideClosure, BreakInsideCoroutine, BreakNonLoop, ConstContinueBadLabel,
+    ContinueLabeledBlock, OutsideLoop, OutsideLoopSuggestion, UnlabeledCfInWhileCondition,
+    UnlabeledInLabeledBlock,
 };
 
 /// The context in which a block is encountered.
@@ -37,6 +40,11 @@ enum Context {
     AnonConst,
     /// E.g. `const { ... }`.
     ConstBlock,
+    /// E.g. `#[loop_match] loop { state = 'label: { /* ... */ } }`.
+    LoopMatch {
+        /// The label of the labeled block (not of the loop itself).
+        labeled_block: Label,
+    },
 }
 
 #[derive(Clone)]
@@ -141,7 +149,12 @@ impl<'hir> Visitor<'hir> for CheckLoopVisitor<'hir> {
                 }
             }
             hir::ExprKind::Loop(ref b, _, source, _) => {
-                self.with_context(Loop(source), |v| v.visit_block(b));
+                let cx = match self.is_loop_match(e, b) {
+                    Some(labeled_block) => LoopMatch { labeled_block },
+                    None => Loop(source),
+                };
+
+                self.with_context(cx, |v| v.visit_block(b));
             }
             hir::ExprKind::Closure(&hir::Closure {
                 ref fn_decl, body, fn_decl_span, kind, ..
@@ -197,6 +210,23 @@ impl<'hir> Visitor<'hir> for CheckLoopVisitor<'hir> {
                     Err(hir::LoopIdError::UnresolvedLabel) => None,
                 };
 
+                // A `#[const_continue]` must break to a block in a `#[loop_match]`.
+                if find_attr!(self.tcx.hir_attrs(e.hir_id), AttributeKind::ConstContinue(_)) {
+                    if let Some(break_label) = break_label.label {
+                        let is_target_label = |cx: &Context| match cx {
+                            Context::LoopMatch { labeled_block } => {
+                                break_label.ident.name == labeled_block.ident.name
+                            }
+                            _ => false,
+                        };
+
+                        if !self.cx_stack.iter().rev().any(is_target_label) {
+                            let span = break_label.ident.span;
+                            self.tcx.dcx().emit_fatal(ConstContinueBadLabel { span });
+                        }
+                    }
+                }
+
                 if let Some(Node::Block(_)) = loop_id.map(|id| self.tcx.hir_node(id)) {
                     return;
                 }
@@ -299,7 +329,7 @@ impl<'hir> CheckLoopVisitor<'hir> {
         cx_pos: usize,
     ) {
         match self.cx_stack[cx_pos] {
-            LabeledBlock | Loop(_) => {}
+            LabeledBlock | Loop(_) | LoopMatch { .. } => {}
             Closure(closure_span) => {
                 self.tcx.dcx().emit_err(BreakInsideClosure {
                     span,
@@ -380,4 +410,36 @@ impl<'hir> CheckLoopVisitor<'hir> {
             });
         }
     }
+
+    /// Is this a loop annotated with `#[loop_match]` that looks syntactically sound?
+    fn is_loop_match(
+        &self,
+        e: &'hir hir::Expr<'hir>,
+        body: &'hir hir::Block<'hir>,
+    ) -> Option<Label> {
+        if !find_attr!(self.tcx.hir_attrs(e.hir_id), AttributeKind::LoopMatch(_)) {
+            return None;
+        }
+
+        // NOTE: Diagnostics are emitted during MIR construction.
+
+        // Accept either `state = expr` or `state = expr;`.
+        let loop_body_expr = match body.stmts {
+            [] => match body.expr {
+                Some(expr) => expr,
+                None => return None,
+            },
+            [single] if body.expr.is_none() => match single.kind {
+                hir::StmtKind::Expr(expr) | hir::StmtKind::Semi(expr) => expr,
+                _ => return None,
+            },
+            [..] => return None,
+        };
+
+        let hir::ExprKind::Assign(_, rhs_expr, _) = loop_body_expr.kind else { return None };
+
+        let hir::ExprKind::Block(_, label) = rhs_expr.kind else { return None };
+
+        label
+    }
 }
diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index d62bf7f85e0..c46e879b976 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -18,7 +18,6 @@ use rustc_parse::parser::attr::AllowLeadingUnsafe;
 use rustc_query_impl::QueryCtxt;
 use rustc_query_system::query::print_query_stack;
 use rustc_session::config::{self, Cfg, CheckCfg, ExpectedValues, Input, OutFileName};
-use rustc_session::filesearch::sysroot_with_fallback;
 use rustc_session::parse::ParseSess;
 use rustc_session::{CompilerIO, EarlyDiagCtxt, Session, lint};
 use rustc_span::source_map::{FileLoader, RealFileLoader, SourceMapInputs};
@@ -405,8 +404,11 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se
 
     crate::callbacks::setup_callbacks();
 
-    let sysroot = config.opts.sysroot.clone();
-    let target = config::build_target_config(&early_dcx, &config.opts.target_triple, &sysroot);
+    let target = config::build_target_config(
+        &early_dcx,
+        &config.opts.target_triple,
+        config.opts.sysroot.path(),
+    );
     let file_loader = config.file_loader.unwrap_or_else(|| Box::new(RealFileLoader));
     let path_mapping = config.opts.file_path_mapping();
     let hash_kind = config.opts.unstable_opts.src_hash_algorithm(&target);
@@ -426,7 +428,7 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se
             let codegen_backend = match config.make_codegen_backend {
                 None => util::get_codegen_backend(
                     &early_dcx,
-                    &sysroot,
+                    &config.opts.sysroot,
                     config.opts.unstable_opts.codegen_backend.as_deref(),
                     &target,
                 ),
@@ -440,7 +442,7 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se
             let temps_dir = config.opts.unstable_opts.temps_dir.as_deref().map(PathBuf::from);
 
             let bundle = match rustc_errors::fluent_bundle(
-                sysroot_with_fallback(&config.opts.sysroot),
+                &config.opts.sysroot.all_paths().collect::<Vec<_>>(),
                 config.opts.unstable_opts.translate_lang.clone(),
                 config.opts.unstable_opts.translate_additional_ftl.as_deref(),
                 config.opts.unstable_opts.translate_directionality_markers,
@@ -469,7 +471,6 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se
                 locale_resources,
                 config.lint_caps,
                 target,
-                sysroot,
                 util::rustc_version_str().unwrap_or("unknown"),
                 config.ice_file,
                 config.using_internal_features,
diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs
index 0b9facfc7af..edfb05e2ccd 100644
--- a/compiler/rustc_interface/src/passes.rs
+++ b/compiler/rustc_interface/src/passes.rs
@@ -355,9 +355,9 @@ fn print_macro_stats(ecx: &ExtCtxt<'_>) {
             "{prefix} {:<name_w$}{:>uses_w$}{:>lines_w$}{:>avg_lines_w$}{:>bytes_w$}{:>avg_bytes_w$}",
             name,
             thousands::usize_with_underscores(uses),
-            thousands::isize_with_underscores(lines),
+            thousands::usize_with_underscores(lines),
             thousands::f64p1_with_underscores(avg_lines),
-            thousands::isize_with_underscores(bytes),
+            thousands::usize_with_underscores(bytes),
             thousands::f64p1_with_underscores(avg_bytes),
         );
     }
@@ -371,7 +371,7 @@ fn early_lint_checks(tcx: TyCtxt<'_>, (): ()) {
     let mut lint_buffer = resolver.lint_buffer.steal();
 
     if sess.opts.unstable_opts.input_stats {
-        input_stats::print_ast_stats(krate, "POST EXPANSION AST STATS", "ast-stats");
+        input_stats::print_ast_stats(tcx, krate);
     }
 
     // Needs to go *after* expansion to be able to check the results of macro expansion.
diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs
index a0012b04c4f..360b5629e9d 100644
--- a/compiler/rustc_interface/src/tests.rs
+++ b/compiler/rustc_interface/src/tests.rs
@@ -41,9 +41,11 @@ where
 
     let matches = optgroups().parse(args).unwrap();
     let sessopts = build_session_options(&mut early_dcx, &matches);
-    let sysroot = sessopts.sysroot.clone();
-    let target =
-        rustc_session::config::build_target_config(&early_dcx, &sessopts.target_triple, &sysroot);
+    let target = rustc_session::config::build_target_config(
+        &early_dcx,
+        &sessopts.target_triple,
+        sessopts.sysroot.path(),
+    );
     let hash_kind = sessopts.unstable_opts.src_hash_algorithm(&target);
     let checksum_hash_kind = sessopts.unstable_opts.checksum_hash_algorithm();
     let sm_inputs = Some(SourceMapInputs {
@@ -72,7 +74,6 @@ where
             vec![],
             Default::default(),
             target,
-            sysroot,
             "",
             None,
             &USING_INTERNAL_FEATURES,
diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs
index 8a7d6117265..0ca4fcc66ca 100644
--- a/compiler/rustc_interface/src/util.rs
+++ b/compiler/rustc_interface/src/util.rs
@@ -11,7 +11,7 @@ use rustc_data_structures::sync;
 use rustc_metadata::{DylibError, load_symbol_from_dylib};
 use rustc_middle::ty::CurrentGcx;
 use rustc_parse::validate_attr;
-use rustc_session::config::{Cfg, OutFileName, OutputFilenames, OutputTypes, host_tuple};
+use rustc_session::config::{Cfg, OutFileName, OutputFilenames, OutputTypes, Sysroot, host_tuple};
 use rustc_session::lint::{self, BuiltinLintDiag, LintBuffer};
 use rustc_session::output::{CRATE_TYPES, categorize_crate_type};
 use rustc_session::{EarlyDiagCtxt, Session, filesearch};
@@ -305,7 +305,7 @@ fn load_backend_from_dylib(early_dcx: &EarlyDiagCtxt, path: &Path) -> MakeBacken
 /// A name of `None` indicates that the default backend should be used.
 pub fn get_codegen_backend(
     early_dcx: &EarlyDiagCtxt,
-    sysroot: &Path,
+    sysroot: &Sysroot,
     backend_name: Option<&str>,
     target: &Target,
 ) -> Box<dyn CodegenBackend> {
@@ -336,25 +336,24 @@ pub fn get_codegen_backend(
 // This is used for rustdoc, but it uses similar machinery to codegen backend
 // loading, so we leave the code here. It is potentially useful for other tools
 // that want to invoke the rustc binary while linking to rustc as well.
-pub fn rustc_path<'a>() -> Option<&'a Path> {
+pub fn rustc_path<'a>(sysroot: &Sysroot) -> Option<&'a Path> {
     static RUSTC_PATH: OnceLock<Option<PathBuf>> = OnceLock::new();
 
-    const BIN_PATH: &str = env!("RUSTC_INSTALL_BINDIR");
-
-    RUSTC_PATH.get_or_init(|| get_rustc_path_inner(BIN_PATH)).as_deref()
-}
-
-fn get_rustc_path_inner(bin_path: &str) -> Option<PathBuf> {
-    let candidate = filesearch::get_or_default_sysroot()
-        .join(bin_path)
-        .join(if cfg!(target_os = "windows") { "rustc.exe" } else { "rustc" });
-    candidate.exists().then_some(candidate)
+    RUSTC_PATH
+        .get_or_init(|| {
+            let candidate = sysroot
+                .default
+                .join(env!("RUSTC_INSTALL_BINDIR"))
+                .join(if cfg!(target_os = "windows") { "rustc.exe" } else { "rustc" });
+            candidate.exists().then_some(candidate)
+        })
+        .as_deref()
 }
 
 #[allow(rustc::untranslatable_diagnostic)] // FIXME: make this translatable
 fn get_codegen_sysroot(
     early_dcx: &EarlyDiagCtxt,
-    sysroot: &Path,
+    sysroot: &Sysroot,
     backend_name: &str,
 ) -> MakeBackendFn {
     // For now we only allow this function to be called once as it'll dlopen a
@@ -369,10 +368,9 @@ fn get_codegen_sysroot(
     );
 
     let target = host_tuple();
-    let sysroot_candidates = filesearch::sysroot_with_fallback(&sysroot);
 
-    let sysroot = sysroot_candidates
-        .iter()
+    let sysroot = sysroot
+        .all_paths()
         .map(|sysroot| {
             filesearch::make_target_lib_path(sysroot, target).with_file_name("codegen-backends")
         })
@@ -381,8 +379,8 @@ fn get_codegen_sysroot(
             f.exists()
         })
         .unwrap_or_else(|| {
-            let candidates = sysroot_candidates
-                .iter()
+            let candidates = sysroot
+                .all_paths()
                 .map(|p| p.display().to_string())
                 .collect::<Vec<_>>()
                 .join("\n* ");
diff --git a/compiler/rustc_lint/src/builtin.rs b/compiler/rustc_lint/src/builtin.rs
index ac405277c4e..5bbe69c8d65 100644
--- a/compiler/rustc_lint/src/builtin.rs
+++ b/compiler/rustc_lint/src/builtin.rs
@@ -1184,11 +1184,11 @@ impl<'tcx> LateLintPass<'tcx> for UngatedAsyncFnTrackCaller {
         if fn_kind.asyncness().is_async()
             && !cx.tcx.features().async_fn_track_caller()
             // Now, check if the function has the `#[track_caller]` attribute
-            && let Some(attr) = cx.tcx.get_attr(def_id, sym::track_caller)
+            && let Some(attr_span) = find_attr!(cx.tcx.get_all_attrs(def_id), AttributeKind::TrackCaller(span) => *span)
         {
             cx.emit_span_lint(
                 UNGATED_ASYNC_FN_TRACK_CALLER,
-                attr.span(),
+                attr_span,
                 BuiltinUngatedAsyncFnTrackCaller { label: span, session: &cx.tcx.sess },
             );
         }
diff --git a/compiler/rustc_lint/src/context.rs b/compiler/rustc_lint/src/context.rs
index 297b8ef7e76..95663204ec3 100644
--- a/compiler/rustc_lint/src/context.rs
+++ b/compiler/rustc_lint/src/context.rs
@@ -524,6 +524,20 @@ pub trait LintContext {
         });
     }
 
+    /// Emit a lint at `span` from a lazily-constructed lint struct (some type that implements
+    /// `LintDiagnostic`, typically generated by `#[derive(LintDiagnostic)]`).
+    fn emit_span_lint_lazy<S: Into<MultiSpan>, L: for<'a> LintDiagnostic<'a, ()>>(
+        &self,
+        lint: &'static Lint,
+        span: S,
+        decorator: impl FnOnce() -> L,
+    ) {
+        self.opt_span_lint(lint, Some(span), |lint| {
+            let decorator = decorator();
+            decorator.decorate_lint(lint);
+        });
+    }
+
     /// Emit a lint at the appropriate level, with an associated span.
     ///
     /// [`lint_level`]: rustc_middle::lint::lint_level#decorate-signature
diff --git a/compiler/rustc_lint/src/lints.rs b/compiler/rustc_lint/src/lints.rs
index abdf8e3853b..eedb897312e 100644
--- a/compiler/rustc_lint/src/lints.rs
+++ b/compiler/rustc_lint/src/lints.rs
@@ -1353,6 +1353,8 @@ pub(crate) struct NonUpperCaseGlobal<'a> {
     pub name: &'a str,
     #[subdiagnostic]
     pub sub: NonUpperCaseGlobalSub,
+    #[subdiagnostic]
+    pub usages: Vec<NonUpperCaseGlobalSubTool>,
 }
 
 #[derive(Subdiagnostic)]
@@ -1362,14 +1364,29 @@ pub(crate) enum NonUpperCaseGlobalSub {
         #[primary_span]
         span: Span,
     },
-    #[suggestion(lint_suggestion, code = "{replace}", applicability = "maybe-incorrect")]
+    #[suggestion(lint_suggestion, code = "{replace}")]
     Suggestion {
         #[primary_span]
         span: Span,
+        #[applicability]
+        applicability: Applicability,
         replace: String,
     },
 }
 
+#[derive(Subdiagnostic)]
+#[suggestion(
+    lint_suggestion,
+    code = "{replace}",
+    applicability = "machine-applicable",
+    style = "tool-only"
+)]
+pub(crate) struct NonUpperCaseGlobalSubTool {
+    #[primary_span]
+    pub(crate) span: Span,
+    pub(crate) replace: String,
+}
+
 // noop_method_call.rs
 #[derive(LintDiagnostic)]
 #[diag(lint_noop_method_call)]
@@ -3261,7 +3278,7 @@ impl Subdiagnostic for MismatchedLifetimeSyntaxesSuggestion {
                 diag.multipart_suggestion_with_style(
                     fluent::lint_mismatched_lifetime_syntaxes_suggestion_implicit,
                     suggestions,
-                    Applicability::MachineApplicable,
+                    Applicability::MaybeIncorrect,
                     style(tool_only),
                 );
             }
@@ -3276,7 +3293,7 @@ impl Subdiagnostic for MismatchedLifetimeSyntaxesSuggestion {
                 diag.multipart_suggestion_with_style(
                     fluent::lint_mismatched_lifetime_syntaxes_suggestion_mixed,
                     suggestions,
-                    Applicability::MachineApplicable,
+                    Applicability::MaybeIncorrect,
                     style(tool_only),
                 );
             }
@@ -3291,7 +3308,7 @@ impl Subdiagnostic for MismatchedLifetimeSyntaxesSuggestion {
                 diag.multipart_suggestion_with_style(
                     msg,
                     suggestions,
-                    Applicability::MachineApplicable,
+                    Applicability::MaybeIncorrect,
                     style(tool_only),
                 );
             }
diff --git a/compiler/rustc_lint/src/nonstandard_style.rs b/compiler/rustc_lint/src/nonstandard_style.rs
index f39e1506390..97e627f2eb2 100644
--- a/compiler/rustc_lint/src/nonstandard_style.rs
+++ b/compiler/rustc_lint/src/nonstandard_style.rs
@@ -1,9 +1,12 @@
 use rustc_abi::ExternAbi;
 use rustc_attr_data_structures::{AttributeKind, ReprAttr, find_attr};
 use rustc_attr_parsing::AttributeParser;
+use rustc_errors::Applicability;
 use rustc_hir::def::{DefKind, Res};
-use rustc_hir::intravisit::FnKind;
+use rustc_hir::def_id::DefId;
+use rustc_hir::intravisit::{FnKind, Visitor};
 use rustc_hir::{AttrArgs, AttrItem, Attribute, GenericParamKind, PatExprKind, PatKind};
+use rustc_middle::hir::nested_filter::All;
 use rustc_middle::ty;
 use rustc_session::config::CrateType;
 use rustc_session::{declare_lint, declare_lint_pass};
@@ -13,7 +16,7 @@ use {rustc_ast as ast, rustc_hir as hir};
 
 use crate::lints::{
     NonCamelCaseType, NonCamelCaseTypeSub, NonSnakeCaseDiag, NonSnakeCaseDiagSub,
-    NonUpperCaseGlobal, NonUpperCaseGlobalSub,
+    NonUpperCaseGlobal, NonUpperCaseGlobalSub, NonUpperCaseGlobalSubTool,
 };
 use crate::{EarlyContext, EarlyLintPass, LateContext, LateLintPass, LintContext};
 
@@ -493,22 +496,82 @@ declare_lint! {
 declare_lint_pass!(NonUpperCaseGlobals => [NON_UPPER_CASE_GLOBALS]);
 
 impl NonUpperCaseGlobals {
-    fn check_upper_case(cx: &LateContext<'_>, sort: &str, ident: &Ident) {
+    fn check_upper_case(cx: &LateContext<'_>, sort: &str, did: Option<LocalDefId>, ident: &Ident) {
         let name = ident.name.as_str();
         if name.chars().any(|c| c.is_lowercase()) {
             let uc = NonSnakeCase::to_snake_case(name).to_uppercase();
+
+            // If the item is exported, suggesting changing it's name would be breaking-change
+            // and could break users without a "nice" applicable fix, so let's avoid it.
+            let can_change_usages = if let Some(did) = did {
+                !cx.tcx.effective_visibilities(()).is_exported(did)
+            } else {
+                false
+            };
+
             // We cannot provide meaningful suggestions
             // if the characters are in the category of "Lowercase Letter".
             let sub = if *name != uc {
-                NonUpperCaseGlobalSub::Suggestion { span: ident.span, replace: uc }
+                NonUpperCaseGlobalSub::Suggestion {
+                    span: ident.span,
+                    replace: uc.clone(),
+                    applicability: if can_change_usages {
+                        Applicability::MachineApplicable
+                    } else {
+                        Applicability::MaybeIncorrect
+                    },
+                }
             } else {
                 NonUpperCaseGlobalSub::Label { span: ident.span }
             };
-            cx.emit_span_lint(
-                NON_UPPER_CASE_GLOBALS,
-                ident.span,
-                NonUpperCaseGlobal { sort, name, sub },
-            );
+
+            struct UsageCollector<'a, 'tcx> {
+                cx: &'tcx LateContext<'a>,
+                did: DefId,
+                collected: Vec<Span>,
+            }
+
+            impl<'v, 'tcx> Visitor<'v> for UsageCollector<'v, 'tcx> {
+                type NestedFilter = All;
+
+                fn maybe_tcx(&mut self) -> Self::MaybeTyCtxt {
+                    self.cx.tcx
+                }
+
+                fn visit_path(
+                    &mut self,
+                    path: &rustc_hir::Path<'v>,
+                    _id: rustc_hir::HirId,
+                ) -> Self::Result {
+                    if let Some(final_seg) = path.segments.last()
+                        && final_seg.res.opt_def_id() == Some(self.did)
+                    {
+                        self.collected.push(final_seg.ident.span);
+                    }
+                }
+            }
+
+            cx.emit_span_lint_lazy(NON_UPPER_CASE_GLOBALS, ident.span, || {
+                // Compute usages lazily as it can expansive and useless when the lint is allowed.
+                // cf. https://github.com/rust-lang/rust/pull/142645#issuecomment-2993024625
+                let usages = if can_change_usages
+                    && *name != uc
+                    && let Some(did) = did
+                {
+                    let mut usage_collector =
+                        UsageCollector { cx, did: did.to_def_id(), collected: Vec::new() };
+                    cx.tcx.hir_walk_toplevel_module(&mut usage_collector);
+                    usage_collector
+                        .collected
+                        .into_iter()
+                        .map(|span| NonUpperCaseGlobalSubTool { span, replace: uc.clone() })
+                        .collect()
+                } else {
+                    vec![]
+                };
+
+                NonUpperCaseGlobal { sort, name, sub, usages }
+            });
         }
     }
 }
@@ -520,10 +583,20 @@ impl<'tcx> LateLintPass<'tcx> for NonUpperCaseGlobals {
             hir::ItemKind::Static(_, ident, ..)
                 if !find_attr!(attrs, AttributeKind::NoMangle(..)) =>
             {
-                NonUpperCaseGlobals::check_upper_case(cx, "static variable", &ident);
+                NonUpperCaseGlobals::check_upper_case(
+                    cx,
+                    "static variable",
+                    Some(it.owner_id.def_id),
+                    &ident,
+                );
             }
             hir::ItemKind::Const(ident, ..) => {
-                NonUpperCaseGlobals::check_upper_case(cx, "constant", &ident);
+                NonUpperCaseGlobals::check_upper_case(
+                    cx,
+                    "constant",
+                    Some(it.owner_id.def_id),
+                    &ident,
+                );
             }
             _ => {}
         }
@@ -531,7 +604,7 @@ impl<'tcx> LateLintPass<'tcx> for NonUpperCaseGlobals {
 
     fn check_trait_item(&mut self, cx: &LateContext<'_>, ti: &hir::TraitItem<'_>) {
         if let hir::TraitItemKind::Const(..) = ti.kind {
-            NonUpperCaseGlobals::check_upper_case(cx, "associated constant", &ti.ident);
+            NonUpperCaseGlobals::check_upper_case(cx, "associated constant", None, &ti.ident);
         }
     }
 
@@ -539,7 +612,7 @@ impl<'tcx> LateLintPass<'tcx> for NonUpperCaseGlobals {
         if let hir::ImplItemKind::Const(..) = ii.kind
             && !assoc_item_in_trait_impl(cx, ii)
         {
-            NonUpperCaseGlobals::check_upper_case(cx, "associated constant", &ii.ident);
+            NonUpperCaseGlobals::check_upper_case(cx, "associated constant", None, &ii.ident);
         }
     }
 
@@ -555,6 +628,7 @@ impl<'tcx> LateLintPass<'tcx> for NonUpperCaseGlobals {
                     NonUpperCaseGlobals::check_upper_case(
                         cx,
                         "constant in pattern",
+                        None,
                         &segment.ident,
                     );
                 }
@@ -564,7 +638,12 @@ impl<'tcx> LateLintPass<'tcx> for NonUpperCaseGlobals {
 
     fn check_generic_param(&mut self, cx: &LateContext<'_>, param: &hir::GenericParam<'_>) {
         if let GenericParamKind::Const { .. } = param.kind {
-            NonUpperCaseGlobals::check_upper_case(cx, "const parameter", &param.name.ident());
+            NonUpperCaseGlobals::check_upper_case(
+                cx,
+                "const parameter",
+                Some(param.def_id),
+                &param.name.ident(),
+            );
         }
     }
 }
diff --git a/compiler/rustc_metadata/src/locator.rs b/compiler/rustc_metadata/src/locator.rs
index 259bcb1b96d..941f16bd960 100644
--- a/compiler/rustc_metadata/src/locator.rs
+++ b/compiler/rustc_metadata/src/locator.rs
@@ -321,7 +321,7 @@ impl<'a> CrateLocator<'a> {
 
         CrateLocator {
             only_needs_metadata,
-            sysroot: &sess.sysroot,
+            sysroot: sess.opts.sysroot.path(),
             metadata_loader,
             cfg_version: sess.cfg_version,
             crate_name,
diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs
index 00bd32eb0eb..ed3c18a02a6 100644
--- a/compiler/rustc_metadata/src/rmeta/encoder.rs
+++ b/compiler/rustc_metadata/src/rmeta/encoder.rs
@@ -762,6 +762,8 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         assert_eq!(total_bytes, computed_total_bytes);
 
         if tcx.sess.opts.unstable_opts.meta_stats {
+            use std::fmt::Write;
+
             self.opaque.flush();
 
             // Rewind and re-read all the metadata to count the zero bytes we wrote.
@@ -777,31 +779,44 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             assert_eq!(self.opaque.file().stream_position().unwrap(), pos_before_rewind);
 
             stats.sort_by_key(|&(_, usize)| usize);
+            stats.reverse(); // bigger items first
 
             let prefix = "meta-stats";
             let perc = |bytes| (bytes * 100) as f64 / total_bytes as f64;
 
-            eprintln!("{prefix} METADATA STATS");
-            eprintln!("{} {:<23}{:>10}", prefix, "Section", "Size");
-            eprintln!("{prefix} ----------------------------------------------------------------");
+            let section_w = 23;
+            let size_w = 10;
+            let banner_w = 64;
+
+            // We write all the text into a string and print it with a single
+            // `eprint!`. This is an attempt to minimize interleaved text if multiple
+            // rustc processes are printing macro-stats at the same time (e.g. with
+            // `RUSTFLAGS='-Zmeta-stats' cargo build`). It still doesn't guarantee
+            // non-interleaving, though.
+            let mut s = String::new();
+            _ = writeln!(s, "{prefix} {}", "=".repeat(banner_w));
+            _ = writeln!(s, "{prefix} METADATA STATS: {}", tcx.crate_name(LOCAL_CRATE));
+            _ = writeln!(s, "{prefix} {:<section_w$}{:>size_w$}", "Section", "Size");
+            _ = writeln!(s, "{prefix} {}", "-".repeat(banner_w));
             for (label, size) in stats {
-                eprintln!(
-                    "{} {:<23}{:>10} ({:4.1}%)",
-                    prefix,
+                _ = writeln!(
+                    s,
+                    "{prefix} {:<section_w$}{:>size_w$} ({:4.1}%)",
                     label,
                     usize_with_underscores(size),
                     perc(size)
                 );
             }
-            eprintln!("{prefix} ----------------------------------------------------------------");
-            eprintln!(
-                "{} {:<23}{:>10} (of which {:.1}% are zero bytes)",
-                prefix,
+            _ = writeln!(s, "{prefix} {}", "-".repeat(banner_w));
+            _ = writeln!(
+                s,
+                "{prefix} {:<section_w$}{:>size_w$} (of which {:.1}% are zero bytes)",
                 "Total",
                 usize_with_underscores(total_bytes),
                 perc(zero_bytes)
             );
-            eprintln!("{prefix}");
+            _ = writeln!(s, "{prefix} {}", "=".repeat(banner_w));
+            eprint!("{s}");
         }
 
         root
diff --git a/compiler/rustc_middle/src/middle/exported_symbols.rs b/compiler/rustc_middle/src/middle/exported_symbols.rs
index 1d67d0fe3bb..64a1f2aff15 100644
--- a/compiler/rustc_middle/src/middle/exported_symbols.rs
+++ b/compiler/rustc_middle/src/middle/exported_symbols.rs
@@ -22,7 +22,7 @@ impl SymbolExportLevel {
 }
 
 /// Kind of exported symbols.
-#[derive(Eq, PartialEq, Debug, Copy, Clone, Encodable, Decodable, HashStable)]
+#[derive(Eq, PartialEq, Debug, Copy, Clone, Encodable, Decodable, HashStable, Hash)]
 pub enum SymbolExportKind {
     Text,
     Data,
diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs
index b9a014d14c0..d0e72a86d8a 100644
--- a/compiler/rustc_middle/src/thir.rs
+++ b/compiler/rustc_middle/src/thir.rs
@@ -378,6 +378,14 @@ pub enum ExprKind<'tcx> {
     Loop {
         body: ExprId,
     },
+    /// A `#[loop_match] loop { state = 'blk: { match state { ... } } }` expression.
+    LoopMatch {
+        /// The state variable that is updated, and also the scrutinee of the match.
+        state: ExprId,
+        region_scope: region::Scope,
+        arms: Box<[ArmId]>,
+        match_span: Span,
+    },
     /// Special expression representing the `let` part of an `if let` or similar construct
     /// (including `if let` guards in match arms, and let-chains formed by `&&`).
     ///
@@ -454,6 +462,11 @@ pub enum ExprKind<'tcx> {
     Continue {
         label: region::Scope,
     },
+    /// A `#[const_continue] break` expression.
+    ConstContinue {
+        label: region::Scope,
+        value: ExprId,
+    },
     /// A `return` expression.
     Return {
         value: Option<ExprId>,
diff --git a/compiler/rustc_middle/src/thir/visit.rs b/compiler/rustc_middle/src/thir/visit.rs
index d8743814d79..c9ef723aea4 100644
--- a/compiler/rustc_middle/src/thir/visit.rs
+++ b/compiler/rustc_middle/src/thir/visit.rs
@@ -83,7 +83,7 @@ pub fn walk_expr<'thir, 'tcx: 'thir, V: Visitor<'thir, 'tcx>>(
             visitor.visit_pat(pat);
         }
         Loop { body } => visitor.visit_expr(&visitor.thir()[body]),
-        Match { scrutinee, ref arms, .. } => {
+        LoopMatch { state: scrutinee, ref arms, .. } | Match { scrutinee, ref arms, .. } => {
             visitor.visit_expr(&visitor.thir()[scrutinee]);
             for &arm in &**arms {
                 visitor.visit_arm(&visitor.thir()[arm]);
@@ -108,6 +108,7 @@ pub fn walk_expr<'thir, 'tcx: 'thir, V: Visitor<'thir, 'tcx>>(
             }
         }
         Continue { label: _ } => {}
+        ConstContinue { value, label: _ } => visitor.visit_expr(&visitor.thir()[value]),
         Return { value } => {
             if let Some(value) = value {
                 visitor.visit_expr(&visitor.thir()[value])
diff --git a/compiler/rustc_mir_build/Cargo.toml b/compiler/rustc_mir_build/Cargo.toml
index 1534865cdd3..c4c2d8a7ac8 100644
--- a/compiler/rustc_mir_build/Cargo.toml
+++ b/compiler/rustc_mir_build/Cargo.toml
@@ -11,6 +11,7 @@ rustc_abi = { path = "../rustc_abi" }
 rustc_apfloat = "0.2.0"
 rustc_arena = { path = "../rustc_arena" }
 rustc_ast = { path = "../rustc_ast" }
+rustc_attr_data_structures = { path = "../rustc_attr_data_structures" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
diff --git a/compiler/rustc_mir_build/messages.ftl b/compiler/rustc_mir_build/messages.ftl
index fae159103e7..e339520cd86 100644
--- a/compiler/rustc_mir_build/messages.ftl
+++ b/compiler/rustc_mir_build/messages.ftl
@@ -84,6 +84,15 @@ mir_build_call_to_unsafe_fn_requires_unsafe_unsafe_op_in_unsafe_fn_allowed =
 
 mir_build_confused = missing patterns are not covered because `{$variable}` is interpreted as a constant pattern, not a new variable
 
+mir_build_const_continue_bad_const = could not determine the target branch for this `#[const_continue]`
+    .label = this value is too generic
+    .note = the value must be a literal or a monomorphic const
+
+mir_build_const_continue_missing_value = a `#[const_continue]` must break to a label with a value
+
+mir_build_const_continue_unknown_jump_target = the target of this `#[const_continue]` is not statically known
+    .label = this value must be a literal or a monomorphic const
+
 mir_build_const_defined_here = constant defined here
 
 mir_build_const_param_in_pattern = constant parameters cannot be referenced in patterns
@@ -212,6 +221,30 @@ mir_build_literal_in_range_out_of_bounds =
     literal out of range for `{$ty}`
     .label = this value does not fit into the type `{$ty}` whose range is `{$min}..={$max}`
 
+mir_build_loop_match_arm_with_guard =
+    match arms that are part of a `#[loop_match]` cannot have guards
+
+mir_build_loop_match_bad_rhs =
+    this expression must be a single `match` wrapped in a labeled block
+
+mir_build_loop_match_bad_statements =
+    statements are not allowed in this position within a `#[loop_match]`
+
+mir_build_loop_match_invalid_match =
+    invalid match on `#[loop_match]` state
+    .note = a local variable must be the scrutinee within a `#[loop_match]`
+
+mir_build_loop_match_invalid_update =
+    invalid update of the `#[loop_match]` state
+    .label = the assignment must update this variable
+
+mir_build_loop_match_missing_assignment =
+    expected a single assignment expression
+
+mir_build_loop_match_unsupported_type =
+    this `#[loop_match]` state value has type `{$ty}`, which is not supported
+    .note = only integers, floats, bool, char, and enums without fields are supported
+
 mir_build_lower_range_bound_must_be_less_than_or_equal_to_upper =
     lower range bound must be less than or equal to upper
     .label = lower bound larger than upper bound
diff --git a/compiler/rustc_mir_build/src/builder/expr/as_place.rs b/compiler/rustc_mir_build/src/builder/expr/as_place.rs
index f8c64d7d13e..99148504a87 100644
--- a/compiler/rustc_mir_build/src/builder/expr/as_place.rs
+++ b/compiler/rustc_mir_build/src/builder/expr/as_place.rs
@@ -565,12 +565,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             | ExprKind::Match { .. }
             | ExprKind::If { .. }
             | ExprKind::Loop { .. }
+            | ExprKind::LoopMatch { .. }
             | ExprKind::Block { .. }
             | ExprKind::Let { .. }
             | ExprKind::Assign { .. }
             | ExprKind::AssignOp { .. }
             | ExprKind::Break { .. }
             | ExprKind::Continue { .. }
+            | ExprKind::ConstContinue { .. }
             | ExprKind::Return { .. }
             | ExprKind::Become { .. }
             | ExprKind::Literal { .. }
diff --git a/compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs b/compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs
index b23bc089cd4..9e07dd5da7e 100644
--- a/compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs
+++ b/compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs
@@ -538,6 +538,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             | ExprKind::RawBorrow { .. }
             | ExprKind::Adt { .. }
             | ExprKind::Loop { .. }
+            | ExprKind::LoopMatch { .. }
             | ExprKind::LogicalOp { .. }
             | ExprKind::Call { .. }
             | ExprKind::Field { .. }
@@ -548,6 +549,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             | ExprKind::UpvarRef { .. }
             | ExprKind::Break { .. }
             | ExprKind::Continue { .. }
+            | ExprKind::ConstContinue { .. }
             | ExprKind::Return { .. }
             | ExprKind::Become { .. }
             | ExprKind::InlineAsm { .. }
diff --git a/compiler/rustc_mir_build/src/builder/expr/category.rs b/compiler/rustc_mir_build/src/builder/expr/category.rs
index 34524aed406..5e4219dbf5b 100644
--- a/compiler/rustc_mir_build/src/builder/expr/category.rs
+++ b/compiler/rustc_mir_build/src/builder/expr/category.rs
@@ -83,9 +83,11 @@ impl Category {
             | ExprKind::NamedConst { .. } => Some(Category::Constant),
 
             ExprKind::Loop { .. }
+            | ExprKind::LoopMatch { .. }
             | ExprKind::Block { .. }
             | ExprKind::Break { .. }
             | ExprKind::Continue { .. }
+            | ExprKind::ConstContinue { .. }
             | ExprKind::Return { .. }
             | ExprKind::Become { .. } =>
             // FIXME(#27840) these probably want their own
diff --git a/compiler/rustc_mir_build/src/builder/expr/into.rs b/compiler/rustc_mir_build/src/builder/expr/into.rs
index 2074fbce0ae..fe3d072fa88 100644
--- a/compiler/rustc_mir_build/src/builder/expr/into.rs
+++ b/compiler/rustc_mir_build/src/builder/expr/into.rs
@@ -8,15 +8,16 @@ use rustc_hir::lang_items::LangItem;
 use rustc_middle::mir::*;
 use rustc_middle::span_bug;
 use rustc_middle::thir::*;
-use rustc_middle::ty::{CanonicalUserTypeAnnotation, Ty};
+use rustc_middle::ty::{self, CanonicalUserTypeAnnotation, Ty};
 use rustc_span::DUMMY_SP;
 use rustc_span::source_map::Spanned;
 use rustc_trait_selection::infer::InferCtxtExt;
 use tracing::{debug, instrument};
 
 use crate::builder::expr::category::{Category, RvalueFunc};
-use crate::builder::matches::DeclareLetBindings;
+use crate::builder::matches::{DeclareLetBindings, HasMatchGuard};
 use crate::builder::{BlockAnd, BlockAndExtension, BlockFrame, Builder, NeedsTemporary};
+use crate::errors::{LoopMatchArmWithGuard, LoopMatchUnsupportedType};
 
 impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Compile `expr`, storing the result into `destination`, which
@@ -244,6 +245,122 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     None
                 })
             }
+            ExprKind::LoopMatch { state, region_scope, match_span, ref arms } => {
+                // Intuitively, this is a combination of a loop containing a labeled block
+                // containing a match.
+                //
+                // The only new bit here is that the lowering of the match is wrapped in a
+                // `in_const_continuable_scope`, which makes the match arms and their target basic
+                // block available to the lowering of `#[const_continue]`.
+
+                fn is_supported_loop_match_type(ty: Ty<'_>) -> bool {
+                    match ty.kind() {
+                        ty::Uint(_) | ty::Int(_) | ty::Float(_) | ty::Bool | ty::Char => true,
+                        ty::Adt(adt_def, _) => match adt_def.adt_kind() {
+                            ty::AdtKind::Struct | ty::AdtKind::Union => false,
+                            ty::AdtKind::Enum => {
+                                adt_def.variants().iter().all(|v| v.fields.is_empty())
+                            }
+                        },
+                        _ => false,
+                    }
+                }
+
+                let state_ty = this.thir.exprs[state].ty;
+                if !is_supported_loop_match_type(state_ty) {
+                    let span = this.thir.exprs[state].span;
+                    this.tcx.dcx().emit_fatal(LoopMatchUnsupportedType { span, ty: state_ty })
+                }
+
+                let loop_block = this.cfg.start_new_block();
+
+                // Start the loop.
+                this.cfg.goto(block, source_info, loop_block);
+
+                this.in_breakable_scope(Some(loop_block), destination, expr_span, |this| {
+                    // Logic for `loop`.
+                    let mut body_block = this.cfg.start_new_block();
+                    this.cfg.terminate(
+                        loop_block,
+                        source_info,
+                        TerminatorKind::FalseUnwind {
+                            real_target: body_block,
+                            unwind: UnwindAction::Continue,
+                        },
+                    );
+                    this.diverge_from(loop_block);
+
+                    // Logic for `match`.
+                    let scrutinee_place_builder =
+                        unpack!(body_block = this.as_place_builder(body_block, state));
+                    let scrutinee_span = this.thir.exprs[state].span;
+                    let match_start_span = match_span.shrink_to_lo().to(scrutinee_span);
+
+                    let mut patterns = Vec::with_capacity(arms.len());
+                    for &arm_id in arms.iter() {
+                        let arm = &this.thir[arm_id];
+
+                        if let Some(guard) = arm.guard {
+                            let span = this.thir.exprs[guard].span;
+                            this.tcx.dcx().emit_fatal(LoopMatchArmWithGuard { span })
+                        }
+
+                        patterns.push((&*arm.pattern, HasMatchGuard::No));
+                    }
+
+                    // The `built_tree` maps match arms to their basic block (where control flow
+                    // jumps to when a value matches the arm). This structure is stored so that a
+                    // `#[const_continue]` can figure out what basic block to jump to.
+                    let built_tree = this.lower_match_tree(
+                        body_block,
+                        scrutinee_span,
+                        &scrutinee_place_builder,
+                        match_start_span,
+                        patterns,
+                        false,
+                    );
+
+                    let state_place = scrutinee_place_builder.to_place(this);
+
+                    // This is logic for the labeled block: a block is a drop scope, hence
+                    // `in_scope`, and a labeled block can be broken out of with a `break 'label`,
+                    // hence the `in_breakable_scope`.
+                    //
+                    // Then `in_const_continuable_scope` stores information for the lowering of
+                    // `#[const_continue]`, and finally the match is lowered in the standard way.
+                    unpack!(
+                        body_block = this.in_scope(
+                            (region_scope, source_info),
+                            LintLevel::Inherited,
+                            move |this| {
+                                this.in_breakable_scope(None, state_place, expr_span, |this| {
+                                    Some(this.in_const_continuable_scope(
+                                        arms.clone(),
+                                        built_tree.clone(),
+                                        state_place,
+                                        expr_span,
+                                        |this| {
+                                            this.lower_match_arms(
+                                                destination,
+                                                scrutinee_place_builder,
+                                                scrutinee_span,
+                                                arms,
+                                                built_tree,
+                                                this.source_info(match_span),
+                                            )
+                                        },
+                                    ))
+                                })
+                            }
+                        )
+                    );
+
+                    this.cfg.goto(body_block, source_info, loop_block);
+
+                    // Loops are only exited by `break` expressions.
+                    None
+                })
+            }
             ExprKind::Call { ty: _, fun, ref args, from_hir_call, fn_span } => {
                 let fun = unpack!(block = this.as_local_operand(block, fun));
                 let args: Box<[_]> = args
@@ -601,6 +718,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
 
             ExprKind::Continue { .. }
+            | ExprKind::ConstContinue { .. }
             | ExprKind::Break { .. }
             | ExprKind::Return { .. }
             | ExprKind::Become { .. } => {
diff --git a/compiler/rustc_mir_build/src/builder/expr/stmt.rs b/compiler/rustc_mir_build/src/builder/expr/stmt.rs
index 2dff26f02f3..675beceea14 100644
--- a/compiler/rustc_mir_build/src/builder/expr/stmt.rs
+++ b/compiler/rustc_mir_build/src/builder/expr/stmt.rs
@@ -98,6 +98,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             ExprKind::Break { label, value } => {
                 this.break_scope(block, value, BreakableTarget::Break(label), source_info)
             }
+            ExprKind::ConstContinue { label, value } => {
+                this.break_const_continuable_scope(block, value, label, source_info)
+            }
             ExprKind::Return { value } => {
                 this.break_scope(block, value, BreakableTarget::Return, source_info)
             }
diff --git a/compiler/rustc_mir_build/src/builder/matches/mod.rs b/compiler/rustc_mir_build/src/builder/matches/mod.rs
index 977d4f3e931..270a7d4b154 100644
--- a/compiler/rustc_mir_build/src/builder/matches/mod.rs
+++ b/compiler/rustc_mir_build/src/builder/matches/mod.rs
@@ -18,7 +18,9 @@ use rustc_middle::bug;
 use rustc_middle::middle::region;
 use rustc_middle::mir::{self, *};
 use rustc_middle::thir::{self, *};
-use rustc_middle::ty::{self, CanonicalUserTypeAnnotation, Ty};
+use rustc_middle::ty::{self, CanonicalUserTypeAnnotation, Ty, ValTree, ValTreeKind};
+use rustc_pattern_analysis::constructor::RangeEnd;
+use rustc_pattern_analysis::rustc::{DeconstructedPat, RustcPatCtxt};
 use rustc_span::{BytePos, Pos, Span, Symbol, sym};
 use tracing::{debug, instrument};
 
@@ -426,7 +428,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// (by [Builder::lower_match_tree]).
     ///
     /// `outer_source_info` is the SourceInfo for the whole match.
-    fn lower_match_arms(
+    pub(crate) fn lower_match_arms(
         &mut self,
         destination: Place<'tcx>,
         scrutinee_place_builder: PlaceBuilder<'tcx>,
@@ -1395,7 +1397,7 @@ pub(crate) struct ArmHasGuard(pub(crate) bool);
 /// A sub-branch in the output of match lowering. Match lowering has generated MIR code that will
 /// branch to `success_block` when the matched value matches the corresponding pattern. If there is
 /// a guard, its failure must continue to `otherwise_block`, which will resume testing patterns.
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 struct MatchTreeSubBranch<'tcx> {
     span: Span,
     /// The block that is branched to if the corresponding subpattern matches.
@@ -1411,7 +1413,7 @@ struct MatchTreeSubBranch<'tcx> {
 }
 
 /// A branch in the output of match lowering.
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 struct MatchTreeBranch<'tcx> {
     sub_branches: Vec<MatchTreeSubBranch<'tcx>>,
 }
@@ -1430,8 +1432,8 @@ struct MatchTreeBranch<'tcx> {
 /// Here the first arm gives the first `MatchTreeBranch`, which has two sub-branches, one for each
 /// alternative of the or-pattern. They are kept separate because each needs to bind `x` to a
 /// different place.
-#[derive(Debug)]
-struct BuiltMatchTree<'tcx> {
+#[derive(Debug, Clone)]
+pub(crate) struct BuiltMatchTree<'tcx> {
     branches: Vec<MatchTreeBranch<'tcx>>,
     otherwise_block: BasicBlock,
     /// If any of the branches had a guard, we collect here the places and locals to fakely borrow
@@ -1489,7 +1491,7 @@ impl<'tcx> MatchTreeBranch<'tcx> {
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
-enum HasMatchGuard {
+pub(crate) enum HasMatchGuard {
     Yes,
     No,
 }
@@ -1504,7 +1506,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// `refutable` indicates whether the candidate list is refutable (for `if let` and `let else`)
     /// or not (for `let` and `match`). In the refutable case we return the block to which we branch
     /// on failure.
-    fn lower_match_tree(
+    pub(crate) fn lower_match_tree(
         &mut self,
         block: BasicBlock,
         scrutinee_span: Span,
@@ -1890,7 +1892,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         debug!("expanding or-pattern: candidate={:#?}\npats={:#?}", candidate, pats);
         candidate.or_span = Some(match_pair.pattern_span);
         candidate.subcandidates = pats
-            .into_vec()
             .into_iter()
             .map(|flat_pat| Candidate::from_flat_pat(flat_pat, candidate.has_guard))
             .collect();
@@ -2864,4 +2865,129 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
         true
     }
+
+    /// Attempt to statically pick the `BasicBlock` that a value would resolve to at runtime.
+    pub(crate) fn static_pattern_match(
+        &self,
+        cx: &RustcPatCtxt<'_, 'tcx>,
+        valtree: ValTree<'tcx>,
+        arms: &[ArmId],
+        built_match_tree: &BuiltMatchTree<'tcx>,
+    ) -> Option<BasicBlock> {
+        let it = arms.iter().zip(built_match_tree.branches.iter());
+        for (&arm_id, branch) in it {
+            let pat = cx.lower_pat(&*self.thir.arms[arm_id].pattern);
+
+            // Peel off or-patterns if they exist.
+            if let rustc_pattern_analysis::rustc::Constructor::Or = pat.ctor() {
+                for pat in pat.iter_fields() {
+                    // For top-level or-patterns (the only ones we accept right now), when the
+                    // bindings are the same (e.g. there are none), the sub_branch is stored just
+                    // once.
+                    let sub_branch = branch
+                        .sub_branches
+                        .get(pat.idx)
+                        .or_else(|| branch.sub_branches.last())
+                        .unwrap();
+
+                    match self.static_pattern_match_inner(valtree, &pat.pat) {
+                        true => return Some(sub_branch.success_block),
+                        false => continue,
+                    }
+                }
+            } else if self.static_pattern_match_inner(valtree, &pat) {
+                return Some(branch.sub_branches[0].success_block);
+            }
+        }
+
+        None
+    }
+
+    /// Helper for [`Self::static_pattern_match`], checking whether the value represented by the
+    /// `ValTree` matches the given pattern. This function does not recurse, meaning that it does
+    /// not handle or-patterns, or patterns for types with fields.
+    fn static_pattern_match_inner(
+        &self,
+        valtree: ty::ValTree<'tcx>,
+        pat: &DeconstructedPat<'_, 'tcx>,
+    ) -> bool {
+        use rustc_pattern_analysis::constructor::{IntRange, MaybeInfiniteInt};
+        use rustc_pattern_analysis::rustc::Constructor;
+
+        match pat.ctor() {
+            Constructor::Variant(variant_index) => {
+                let ValTreeKind::Branch(box [actual_variant_idx]) = *valtree else {
+                    bug!("malformed valtree for an enum")
+                };
+
+                let ValTreeKind::Leaf(actual_variant_idx) = ***actual_variant_idx else {
+                    bug!("malformed valtree for an enum")
+                };
+
+                *variant_index == VariantIdx::from_u32(actual_variant_idx.to_u32())
+            }
+            Constructor::IntRange(int_range) => {
+                let size = pat.ty().primitive_size(self.tcx);
+                let actual_int = valtree.unwrap_leaf().to_bits(size);
+                let actual_int = if pat.ty().is_signed() {
+                    MaybeInfiniteInt::new_finite_int(actual_int, size.bits())
+                } else {
+                    MaybeInfiniteInt::new_finite_uint(actual_int)
+                };
+                IntRange::from_singleton(actual_int).is_subrange(int_range)
+            }
+            Constructor::Bool(pattern_value) => match valtree.unwrap_leaf().try_to_bool() {
+                Ok(actual_value) => *pattern_value == actual_value,
+                Err(()) => bug!("bool value with invalid bits"),
+            },
+            Constructor::F16Range(l, h, end) => {
+                let actual = valtree.unwrap_leaf().to_f16();
+                match end {
+                    RangeEnd::Included => (*l..=*h).contains(&actual),
+                    RangeEnd::Excluded => (*l..*h).contains(&actual),
+                }
+            }
+            Constructor::F32Range(l, h, end) => {
+                let actual = valtree.unwrap_leaf().to_f32();
+                match end {
+                    RangeEnd::Included => (*l..=*h).contains(&actual),
+                    RangeEnd::Excluded => (*l..*h).contains(&actual),
+                }
+            }
+            Constructor::F64Range(l, h, end) => {
+                let actual = valtree.unwrap_leaf().to_f64();
+                match end {
+                    RangeEnd::Included => (*l..=*h).contains(&actual),
+                    RangeEnd::Excluded => (*l..*h).contains(&actual),
+                }
+            }
+            Constructor::F128Range(l, h, end) => {
+                let actual = valtree.unwrap_leaf().to_f128();
+                match end {
+                    RangeEnd::Included => (*l..=*h).contains(&actual),
+                    RangeEnd::Excluded => (*l..*h).contains(&actual),
+                }
+            }
+            Constructor::Wildcard => true,
+
+            // These we may eventually support:
+            Constructor::Struct
+            | Constructor::Ref
+            | Constructor::DerefPattern(_)
+            | Constructor::Slice(_)
+            | Constructor::UnionField
+            | Constructor::Or
+            | Constructor::Str(_) => bug!("unsupported pattern constructor {:?}", pat.ctor()),
+
+            // These should never occur here:
+            Constructor::Opaque(_)
+            | Constructor::Never
+            | Constructor::NonExhaustive
+            | Constructor::Hidden
+            | Constructor::Missing
+            | Constructor::PrivateUninhabited => {
+                bug!("unsupported pattern constructor {:?}", pat.ctor())
+            }
+        }
+    }
 }
diff --git a/compiler/rustc_mir_build/src/builder/scope.rs b/compiler/rustc_mir_build/src/builder/scope.rs
index 67988f1fcbc..1d15e7e126f 100644
--- a/compiler/rustc_mir_build/src/builder/scope.rs
+++ b/compiler/rustc_mir_build/src/builder/scope.rs
@@ -83,20 +83,24 @@ that contains only loops and breakable blocks. It tracks where a `break`,
 
 use std::mem;
 
+use interpret::ErrorHandled;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::HirId;
 use rustc_index::{IndexSlice, IndexVec};
 use rustc_middle::middle::region;
-use rustc_middle::mir::*;
-use rustc_middle::thir::{ExprId, LintLevel};
-use rustc_middle::ty::{self, TyCtxt};
+use rustc_middle::mir::{self, *};
+use rustc_middle::thir::{AdtExpr, AdtExprBase, ArmId, ExprId, ExprKind, LintLevel};
+use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitableExt, ValTree};
 use rustc_middle::{bug, span_bug};
+use rustc_pattern_analysis::rustc::RustcPatCtxt;
 use rustc_session::lint::Level;
 use rustc_span::source_map::Spanned;
 use rustc_span::{DUMMY_SP, Span};
 use tracing::{debug, instrument};
 
+use super::matches::BuiltMatchTree;
 use crate::builder::{BlockAnd, BlockAndExtension, BlockFrame, Builder, CFG};
+use crate::errors::{ConstContinueBadConst, ConstContinueUnknownJumpTarget};
 
 #[derive(Debug)]
 pub(crate) struct Scopes<'tcx> {
@@ -105,6 +109,8 @@ pub(crate) struct Scopes<'tcx> {
     /// The current set of breakable scopes. See module comment for more details.
     breakable_scopes: Vec<BreakableScope<'tcx>>,
 
+    const_continuable_scopes: Vec<ConstContinuableScope<'tcx>>,
+
     /// The scope of the innermost if-then currently being lowered.
     if_then_scope: Option<IfThenScope>,
 
@@ -175,6 +181,20 @@ struct BreakableScope<'tcx> {
 }
 
 #[derive(Debug)]
+struct ConstContinuableScope<'tcx> {
+    /// The scope for the `#[loop_match]` which its `#[const_continue]`s will jump to.
+    region_scope: region::Scope,
+    /// The place of the state of a `#[loop_match]`, which a `#[const_continue]` must update.
+    state_place: Place<'tcx>,
+
+    arms: Box<[ArmId]>,
+    built_match_tree: BuiltMatchTree<'tcx>,
+
+    /// Drops that happen on a `#[const_continue]`
+    const_continue_drops: DropTree,
+}
+
+#[derive(Debug)]
 struct IfThenScope {
     /// The if-then scope or arm scope
     region_scope: region::Scope,
@@ -461,6 +481,7 @@ impl<'tcx> Scopes<'tcx> {
         Self {
             scopes: Vec::new(),
             breakable_scopes: Vec::new(),
+            const_continuable_scopes: Vec::new(),
             if_then_scope: None,
             unwind_drops: DropTree::new(),
             coroutine_drops: DropTree::new(),
@@ -552,6 +573,59 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         }
     }
 
+    /// Start a const-continuable scope, which tracks where `#[const_continue] break` should
+    /// branch to.
+    pub(crate) fn in_const_continuable_scope<F>(
+        &mut self,
+        arms: Box<[ArmId]>,
+        built_match_tree: BuiltMatchTree<'tcx>,
+        state_place: Place<'tcx>,
+        span: Span,
+        f: F,
+    ) -> BlockAnd<()>
+    where
+        F: FnOnce(&mut Builder<'a, 'tcx>) -> BlockAnd<()>,
+    {
+        let region_scope = self.scopes.topmost();
+        let scope = ConstContinuableScope {
+            region_scope,
+            state_place,
+            const_continue_drops: DropTree::new(),
+            arms,
+            built_match_tree,
+        };
+        self.scopes.const_continuable_scopes.push(scope);
+        let normal_exit_block = f(self);
+        let const_continue_scope = self.scopes.const_continuable_scopes.pop().unwrap();
+        assert!(const_continue_scope.region_scope == region_scope);
+
+        let break_block = self.build_exit_tree(
+            const_continue_scope.const_continue_drops,
+            region_scope,
+            span,
+            None,
+        );
+
+        match (normal_exit_block, break_block) {
+            (block, None) => block,
+            (normal_block, Some(exit_block)) => {
+                let target = self.cfg.start_new_block();
+                let source_info = self.source_info(span);
+                self.cfg.terminate(
+                    normal_block.into_block(),
+                    source_info,
+                    TerminatorKind::Goto { target },
+                );
+                self.cfg.terminate(
+                    exit_block.into_block(),
+                    source_info,
+                    TerminatorKind::Goto { target },
+                );
+                target.unit()
+            }
+        }
+    }
+
     /// Start an if-then scope which tracks drop for `if` expressions and `if`
     /// guards.
     ///
@@ -742,6 +816,190 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         self.cfg.start_new_block().unit()
     }
 
+    /// Based on `FunctionCx::eval_unevaluated_mir_constant_to_valtree`.
+    fn eval_unevaluated_mir_constant_to_valtree(
+        &self,
+        constant: ConstOperand<'tcx>,
+    ) -> Result<(ty::ValTree<'tcx>, Ty<'tcx>), interpret::ErrorHandled> {
+        assert!(!constant.const_.ty().has_param());
+        let (uv, ty) = match constant.const_ {
+            mir::Const::Unevaluated(uv, ty) => (uv.shrink(), ty),
+            mir::Const::Ty(_, c) => match c.kind() {
+                // A constant that came from a const generic but was then used as an argument to
+                // old-style simd_shuffle (passing as argument instead of as a generic param).
+                ty::ConstKind::Value(cv) => return Ok((cv.valtree, cv.ty)),
+                other => span_bug!(constant.span, "{other:#?}"),
+            },
+            mir::Const::Val(mir::ConstValue::Scalar(mir::interpret::Scalar::Int(val)), ty) => {
+                return Ok((ValTree::from_scalar_int(self.tcx, val), ty));
+            }
+            // We should never encounter `Const::Val` unless MIR opts (like const prop) evaluate
+            // a constant and write that value back into `Operand`s. This could happen, but is
+            // unlikely. Also: all users of `simd_shuffle` are on unstable and already need to take
+            // a lot of care around intrinsics. For an issue to happen here, it would require a
+            // macro expanding to a `simd_shuffle` call without wrapping the constant argument in a
+            // `const {}` block, but the user pass through arbitrary expressions.
+
+            // FIXME(oli-obk): Replace the magic const generic argument of `simd_shuffle` with a
+            // real const generic, and get rid of this entire function.
+            other => span_bug!(constant.span, "{other:#?}"),
+        };
+
+        match self.tcx.const_eval_resolve_for_typeck(self.typing_env(), uv, constant.span) {
+            Ok(Ok(valtree)) => Ok((valtree, ty)),
+            Ok(Err(ty)) => span_bug!(constant.span, "could not convert {ty:?} to a valtree"),
+            Err(e) => Err(e),
+        }
+    }
+
+    /// Sets up the drops for jumping from `block` to `scope`.
+    pub(crate) fn break_const_continuable_scope(
+        &mut self,
+        mut block: BasicBlock,
+        value: ExprId,
+        scope: region::Scope,
+        source_info: SourceInfo,
+    ) -> BlockAnd<()> {
+        let span = source_info.span;
+
+        // A break can only break out of a scope, so the value should be a scope.
+        let rustc_middle::thir::ExprKind::Scope { value, .. } = self.thir[value].kind else {
+            span_bug!(span, "break value must be a scope")
+        };
+
+        let constant = match &self.thir[value].kind {
+            ExprKind::Adt(box AdtExpr { variant_index, fields, base, .. }) => {
+                assert!(matches!(base, AdtExprBase::None));
+                assert!(fields.is_empty());
+                ConstOperand {
+                    span: self.thir[value].span,
+                    user_ty: None,
+                    const_: Const::Ty(
+                        self.thir[value].ty,
+                        ty::Const::new_value(
+                            self.tcx,
+                            ValTree::from_branches(
+                                self.tcx,
+                                [ValTree::from_scalar_int(self.tcx, variant_index.as_u32().into())],
+                            ),
+                            self.thir[value].ty,
+                        ),
+                    ),
+                }
+            }
+            _ => self.as_constant(&self.thir[value]),
+        };
+
+        let break_index = self
+            .scopes
+            .const_continuable_scopes
+            .iter()
+            .rposition(|const_continuable_scope| const_continuable_scope.region_scope == scope)
+            .unwrap_or_else(|| span_bug!(span, "no enclosing const-continuable scope found"));
+
+        let scope = &self.scopes.const_continuable_scopes[break_index];
+
+        let state_decl = &self.local_decls[scope.state_place.as_local().unwrap()];
+        let state_ty = state_decl.ty;
+        let (discriminant_ty, rvalue) = match state_ty.kind() {
+            ty::Adt(adt_def, _) if adt_def.is_enum() => {
+                (state_ty.discriminant_ty(self.tcx), Rvalue::Discriminant(scope.state_place))
+            }
+            ty::Uint(_) | ty::Int(_) | ty::Float(_) | ty::Bool | ty::Char => {
+                (state_ty, Rvalue::Use(Operand::Copy(scope.state_place)))
+            }
+            _ => span_bug!(state_decl.source_info.span, "unsupported #[loop_match] state"),
+        };
+
+        // The `PatCtxt` is normally used in pattern exhaustiveness checking, but reused
+        // here because it performs normalization and const evaluation.
+        let dropless_arena = rustc_arena::DroplessArena::default();
+        let typeck_results = self.tcx.typeck(self.def_id);
+        let cx = RustcPatCtxt {
+            tcx: self.tcx,
+            typeck_results,
+            module: self.tcx.parent_module(self.hir_id).to_def_id(),
+            // FIXME(#132279): We're in a body, should handle opaques.
+            typing_env: rustc_middle::ty::TypingEnv::non_body_analysis(self.tcx, self.def_id),
+            dropless_arena: &dropless_arena,
+            match_lint_level: self.hir_id,
+            whole_match_span: Some(rustc_span::Span::default()),
+            scrut_span: rustc_span::Span::default(),
+            refutable: true,
+            known_valid_scrutinee: true,
+        };
+
+        let valtree = match self.eval_unevaluated_mir_constant_to_valtree(constant) {
+            Ok((valtree, ty)) => {
+                // Defensively check that the type is monomorphic.
+                assert!(!ty.has_param());
+
+                valtree
+            }
+            Err(ErrorHandled::Reported(..)) => return self.cfg.start_new_block().unit(),
+            Err(ErrorHandled::TooGeneric(_)) => {
+                self.tcx.dcx().emit_fatal(ConstContinueBadConst { span: constant.span });
+            }
+        };
+
+        let Some(real_target) =
+            self.static_pattern_match(&cx, valtree, &*scope.arms, &scope.built_match_tree)
+        else {
+            self.tcx.dcx().emit_fatal(ConstContinueUnknownJumpTarget { span })
+        };
+
+        self.block_context.push(BlockFrame::SubExpr);
+        let state_place = scope.state_place;
+        block = self.expr_into_dest(state_place, block, value).into_block();
+        self.block_context.pop();
+
+        let discr = self.temp(discriminant_ty, source_info.span);
+        let scope_index = self
+            .scopes
+            .scope_index(self.scopes.const_continuable_scopes[break_index].region_scope, span);
+        let scope = &mut self.scopes.const_continuable_scopes[break_index];
+        self.cfg.push_assign(block, source_info, discr, rvalue);
+        let drop_and_continue_block = self.cfg.start_new_block();
+        let imaginary_target = self.cfg.start_new_block();
+        self.cfg.terminate(
+            block,
+            source_info,
+            TerminatorKind::FalseEdge { real_target: drop_and_continue_block, imaginary_target },
+        );
+
+        let drops = &mut scope.const_continue_drops;
+
+        let drop_idx = self.scopes.scopes[scope_index + 1..]
+            .iter()
+            .flat_map(|scope| &scope.drops)
+            .fold(ROOT_NODE, |drop_idx, &drop| drops.add_drop(drop, drop_idx));
+
+        drops.add_entry_point(imaginary_target, drop_idx);
+
+        self.cfg.terminate(imaginary_target, source_info, TerminatorKind::UnwindResume);
+
+        let region_scope = scope.region_scope;
+        let scope_index = self.scopes.scope_index(region_scope, span);
+        let mut drops = DropTree::new();
+
+        let drop_idx = self.scopes.scopes[scope_index + 1..]
+            .iter()
+            .flat_map(|scope| &scope.drops)
+            .fold(ROOT_NODE, |drop_idx, &drop| drops.add_drop(drop, drop_idx));
+
+        drops.add_entry_point(drop_and_continue_block, drop_idx);
+
+        // `build_drop_trees` doesn't have access to our source_info, so we
+        // create a dummy terminator now. `TerminatorKind::UnwindResume` is used
+        // because MIR type checking will panic if it hasn't been overwritten.
+        // (See `<ExitScopes as DropTreeBuilder>::link_entry_point`.)
+        self.cfg.terminate(drop_and_continue_block, source_info, TerminatorKind::UnwindResume);
+
+        self.build_exit_tree(drops, region_scope, span, Some(real_target));
+
+        return self.cfg.start_new_block().unit();
+    }
+
     /// Sets up the drops for breaking from `block` due to an `if` condition
     /// that turned out to be false.
     ///
diff --git a/compiler/rustc_mir_build/src/check_unsafety.rs b/compiler/rustc_mir_build/src/check_unsafety.rs
index d5061b71699..0b6b36640e9 100644
--- a/compiler/rustc_mir_build/src/check_unsafety.rs
+++ b/compiler/rustc_mir_build/src/check_unsafety.rs
@@ -465,10 +465,12 @@ impl<'a, 'tcx> Visitor<'a, 'tcx> for UnsafetyVisitor<'a, 'tcx> {
             | ExprKind::Break { .. }
             | ExprKind::Closure { .. }
             | ExprKind::Continue { .. }
+            | ExprKind::ConstContinue { .. }
             | ExprKind::Return { .. }
             | ExprKind::Become { .. }
             | ExprKind::Yield { .. }
             | ExprKind::Loop { .. }
+            | ExprKind::LoopMatch { .. }
             | ExprKind::Let { .. }
             | ExprKind::Match { .. }
             | ExprKind::Box { .. }
diff --git a/compiler/rustc_mir_build/src/errors.rs b/compiler/rustc_mir_build/src/errors.rs
index ae09db50235..32df191cbca 100644
--- a/compiler/rustc_mir_build/src/errors.rs
+++ b/compiler/rustc_mir_build/src/errors.rs
@@ -1149,3 +1149,80 @@ impl Subdiagnostic for Rust2024IncompatiblePatSugg {
         }
     }
 }
+
+#[derive(Diagnostic)]
+#[diag(mir_build_loop_match_invalid_update)]
+pub(crate) struct LoopMatchInvalidUpdate {
+    #[primary_span]
+    pub lhs: Span,
+    #[label]
+    pub scrutinee: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(mir_build_loop_match_invalid_match)]
+#[note]
+pub(crate) struct LoopMatchInvalidMatch {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(mir_build_loop_match_unsupported_type)]
+#[note]
+pub(crate) struct LoopMatchUnsupportedType<'tcx> {
+    #[primary_span]
+    pub span: Span,
+    pub ty: Ty<'tcx>,
+}
+
+#[derive(Diagnostic)]
+#[diag(mir_build_loop_match_bad_statements)]
+pub(crate) struct LoopMatchBadStatements {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(mir_build_loop_match_bad_rhs)]
+pub(crate) struct LoopMatchBadRhs {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(mir_build_loop_match_missing_assignment)]
+pub(crate) struct LoopMatchMissingAssignment {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(mir_build_loop_match_arm_with_guard)]
+pub(crate) struct LoopMatchArmWithGuard {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(mir_build_const_continue_bad_const)]
+pub(crate) struct ConstContinueBadConst {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(mir_build_const_continue_missing_value)]
+pub(crate) struct ConstContinueMissingValue {
+    #[primary_span]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(mir_build_const_continue_unknown_jump_target)]
+#[note]
+pub(crate) struct ConstContinueUnknownJumpTarget {
+    #[primary_span]
+    pub span: Span,
+}
diff --git a/compiler/rustc_mir_build/src/thir/cx/expr.rs b/compiler/rustc_mir_build/src/thir/cx/expr.rs
index 3baeccf6409..5197e93fda7 100644
--- a/compiler/rustc_mir_build/src/thir/cx/expr.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/expr.rs
@@ -1,6 +1,7 @@
 use itertools::Itertools;
 use rustc_abi::{FIRST_VARIANT, FieldIdx};
 use rustc_ast::UnsafeBinderCastKind;
+use rustc_attr_data_structures::{AttributeKind, find_attr};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_hir as hir;
 use rustc_hir::def::{CtorKind, CtorOf, DefKind, Res};
@@ -21,6 +22,7 @@ use rustc_middle::{bug, span_bug};
 use rustc_span::{Span, sym};
 use tracing::{debug, info, instrument, trace};
 
+use crate::errors::*;
 use crate::thir::cx::ThirBuildCx;
 
 impl<'tcx> ThirBuildCx<'tcx> {
@@ -479,6 +481,55 @@ impl<'tcx> ThirBuildCx<'tcx> {
                 ExprKind::RawBorrow { mutability, arg: self.mirror_expr(arg) }
             }
 
+            // Make `&pin mut $expr` and `&pin const $expr` into
+            // `Pin { __pointer: &mut { $expr } }` and `Pin { __pointer: &$expr }`.
+            hir::ExprKind::AddrOf(hir::BorrowKind::Pin, mutbl, arg_expr) => match expr_ty.kind() {
+                &ty::Adt(adt_def, args) if tcx.is_lang_item(adt_def.did(), hir::LangItem::Pin) => {
+                    let ty = args.type_at(0);
+                    let arg_ty = self.typeck_results.expr_ty(arg_expr);
+                    let mut arg = self.mirror_expr(arg_expr);
+                    // For `&pin mut $place` where `$place` is not `Unpin`, move the place
+                    // `$place` to ensure it will not be used afterwards.
+                    if mutbl.is_mut() && !arg_ty.is_unpin(self.tcx, self.typing_env) {
+                        let block = self.thir.blocks.push(Block {
+                            targeted_by_break: false,
+                            region_scope: region::Scope {
+                                local_id: arg_expr.hir_id.local_id,
+                                data: region::ScopeData::Node,
+                            },
+                            span: arg_expr.span,
+                            stmts: Box::new([]),
+                            expr: Some(arg),
+                            safety_mode: BlockSafety::Safe,
+                        });
+                        let (temp_lifetime, backwards_incompatible) = self
+                            .rvalue_scopes
+                            .temporary_scope(self.region_scope_tree, arg_expr.hir_id.local_id);
+                        arg = self.thir.exprs.push(Expr {
+                            temp_lifetime: TempLifetime { temp_lifetime, backwards_incompatible },
+                            ty: arg_ty,
+                            span: arg_expr.span,
+                            kind: ExprKind::Block { block },
+                        });
+                    }
+                    let expr = self.thir.exprs.push(Expr {
+                        temp_lifetime: TempLifetime { temp_lifetime, backwards_incompatible },
+                        ty,
+                        span: expr.span,
+                        kind: ExprKind::Borrow { borrow_kind: mutbl.to_borrow_kind(), arg },
+                    });
+                    ExprKind::Adt(Box::new(AdtExpr {
+                        adt_def,
+                        variant_index: FIRST_VARIANT,
+                        args,
+                        fields: Box::new([FieldExpr { name: FieldIdx::from(0u32), expr }]),
+                        user_ty: None,
+                        base: AdtExprBase::None,
+                    }))
+                }
+                _ => span_bug!(expr.span, "unexpected type for pinned borrow: {:?}", expr_ty),
+            },
+
             hir::ExprKind::Block(blk, _) => ExprKind::Block { block: self.mirror_block(blk) },
 
             hir::ExprKind::Assign(lhs, rhs, _) => {
@@ -796,16 +847,38 @@ impl<'tcx> ThirBuildCx<'tcx> {
             }
             hir::ExprKind::Ret(v) => ExprKind::Return { value: v.map(|v| self.mirror_expr(v)) },
             hir::ExprKind::Become(call) => ExprKind::Become { value: self.mirror_expr(call) },
-            hir::ExprKind::Break(dest, ref value) => match dest.target_id {
-                Ok(target_id) => ExprKind::Break {
-                    label: region::Scope {
-                        local_id: target_id.local_id,
-                        data: region::ScopeData::Node,
-                    },
-                    value: value.map(|value| self.mirror_expr(value)),
-                },
-                Err(err) => bug!("invalid loop id for break: {}", err),
-            },
+            hir::ExprKind::Break(dest, ref value) => {
+                if find_attr!(self.tcx.hir_attrs(expr.hir_id), AttributeKind::ConstContinue(_)) {
+                    match dest.target_id {
+                        Ok(target_id) => {
+                            let Some(value) = value else {
+                                let span = expr.span;
+                                self.tcx.dcx().emit_fatal(ConstContinueMissingValue { span })
+                            };
+
+                            ExprKind::ConstContinue {
+                                label: region::Scope {
+                                    local_id: target_id.local_id,
+                                    data: region::ScopeData::Node,
+                                },
+                                value: self.mirror_expr(value),
+                            }
+                        }
+                        Err(err) => bug!("invalid loop id for break: {}", err),
+                    }
+                } else {
+                    match dest.target_id {
+                        Ok(target_id) => ExprKind::Break {
+                            label: region::Scope {
+                                local_id: target_id.local_id,
+                                data: region::ScopeData::Node,
+                            },
+                            value: value.map(|value| self.mirror_expr(value)),
+                        },
+                        Err(err) => bug!("invalid loop id for break: {}", err),
+                    }
+                }
+            }
             hir::ExprKind::Continue(dest) => match dest.target_id {
                 Ok(loop_id) => ExprKind::Continue {
                     label: region::Scope {
@@ -840,18 +913,93 @@ impl<'tcx> ThirBuildCx<'tcx> {
                 match_source,
             },
             hir::ExprKind::Loop(body, ..) => {
-                let block_ty = self.typeck_results.node_type(body.hir_id);
-                let (temp_lifetime, backwards_incompatible) = self
-                    .rvalue_scopes
-                    .temporary_scope(self.region_scope_tree, body.hir_id.local_id);
-                let block = self.mirror_block(body);
-                let body = self.thir.exprs.push(Expr {
-                    ty: block_ty,
-                    temp_lifetime: TempLifetime { temp_lifetime, backwards_incompatible },
-                    span: self.thir[block].span,
-                    kind: ExprKind::Block { block },
-                });
-                ExprKind::Loop { body }
+                if find_attr!(self.tcx.hir_attrs(expr.hir_id), AttributeKind::LoopMatch(_)) {
+                    let dcx = self.tcx.dcx();
+
+                    // Accept either `state = expr` or `state = expr;`.
+                    let loop_body_expr = match body.stmts {
+                        [] => match body.expr {
+                            Some(expr) => expr,
+                            None => dcx.emit_fatal(LoopMatchMissingAssignment { span: body.span }),
+                        },
+                        [single] if body.expr.is_none() => match single.kind {
+                            hir::StmtKind::Expr(expr) | hir::StmtKind::Semi(expr) => expr,
+                            _ => dcx.emit_fatal(LoopMatchMissingAssignment { span: body.span }),
+                        },
+                        [first @ last] | [first, .., last] => dcx
+                            .emit_fatal(LoopMatchBadStatements { span: first.span.to(last.span) }),
+                    };
+
+                    let hir::ExprKind::Assign(state, rhs_expr, _) = loop_body_expr.kind else {
+                        dcx.emit_fatal(LoopMatchMissingAssignment { span: loop_body_expr.span })
+                    };
+
+                    let hir::ExprKind::Block(block_body, _) = rhs_expr.kind else {
+                        dcx.emit_fatal(LoopMatchBadRhs { span: rhs_expr.span })
+                    };
+
+                    // The labeled block should contain one match expression, but defining items is
+                    // allowed.
+                    for stmt in block_body.stmts {
+                        if !matches!(stmt.kind, rustc_hir::StmtKind::Item(_)) {
+                            dcx.emit_fatal(LoopMatchBadStatements { span: stmt.span })
+                        }
+                    }
+
+                    let Some(block_body_expr) = block_body.expr else {
+                        dcx.emit_fatal(LoopMatchBadRhs { span: block_body.span })
+                    };
+
+                    let hir::ExprKind::Match(scrutinee, arms, _match_source) = block_body_expr.kind
+                    else {
+                        dcx.emit_fatal(LoopMatchBadRhs { span: block_body_expr.span })
+                    };
+
+                    fn local(expr: &rustc_hir::Expr<'_>) -> Option<hir::HirId> {
+                        if let hir::ExprKind::Path(hir::QPath::Resolved(_, path)) = expr.kind {
+                            if let Res::Local(hir_id) = path.res {
+                                return Some(hir_id);
+                            }
+                        }
+
+                        None
+                    }
+
+                    let Some(scrutinee_hir_id) = local(scrutinee) else {
+                        dcx.emit_fatal(LoopMatchInvalidMatch { span: scrutinee.span })
+                    };
+
+                    if local(state) != Some(scrutinee_hir_id) {
+                        dcx.emit_fatal(LoopMatchInvalidUpdate {
+                            scrutinee: scrutinee.span,
+                            lhs: state.span,
+                        })
+                    }
+
+                    ExprKind::LoopMatch {
+                        state: self.mirror_expr(state),
+                        region_scope: region::Scope {
+                            local_id: block_body.hir_id.local_id,
+                            data: region::ScopeData::Node,
+                        },
+
+                        arms: arms.iter().map(|a| self.convert_arm(a)).collect(),
+                        match_span: block_body_expr.span,
+                    }
+                } else {
+                    let block_ty = self.typeck_results.node_type(body.hir_id);
+                    let (temp_lifetime, backwards_incompatible) = self
+                        .rvalue_scopes
+                        .temporary_scope(self.region_scope_tree, body.hir_id.local_id);
+                    let block = self.mirror_block(body);
+                    let body = self.thir.exprs.push(Expr {
+                        ty: block_ty,
+                        temp_lifetime: TempLifetime { temp_lifetime, backwards_incompatible },
+                        span: self.thir[block].span,
+                        kind: ExprKind::Block { block },
+                    });
+                    ExprKind::Loop { body }
+                }
             }
             hir::ExprKind::Field(source, ..) => ExprKind::Field {
                 lhs: self.mirror_expr(source),
diff --git a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
index 245bd866030..9fd410e6bf1 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
@@ -331,7 +331,11 @@ impl<'p, 'tcx> MatchVisitor<'p, 'tcx> {
             | WrapUnsafeBinder { source } => self.is_known_valid_scrutinee(&self.thir()[*source]),
 
             // These diverge.
-            Become { .. } | Break { .. } | Continue { .. } | Return { .. } => true,
+            Become { .. }
+            | Break { .. }
+            | Continue { .. }
+            | ConstContinue { .. }
+            | Return { .. } => true,
 
             // These are statements that evaluate to `()`.
             Assign { .. } | AssignOp { .. } | InlineAsm { .. } | Let { .. } => true,
@@ -353,6 +357,7 @@ impl<'p, 'tcx> MatchVisitor<'p, 'tcx> {
             | Literal { .. }
             | LogicalOp { .. }
             | Loop { .. }
+            | LoopMatch { .. }
             | Match { .. }
             | NamedConst { .. }
             | NonHirLiteral { .. }
diff --git a/compiler/rustc_mir_build/src/thir/print.rs b/compiler/rustc_mir_build/src/thir/print.rs
index db9547a481f..1507b6b8c06 100644
--- a/compiler/rustc_mir_build/src/thir/print.rs
+++ b/compiler/rustc_mir_build/src/thir/print.rs
@@ -318,6 +318,20 @@ impl<'a, 'tcx> ThirPrinter<'a, 'tcx> {
                 self.print_expr(*body, depth_lvl + 2);
                 print_indented!(self, ")", depth_lvl);
             }
+            LoopMatch { state, region_scope, match_span, arms } => {
+                print_indented!(self, "LoopMatch {", depth_lvl);
+                print_indented!(self, "state:", depth_lvl + 1);
+                self.print_expr(*state, depth_lvl + 2);
+                print_indented!(self, format!("region_scope: {:?}", region_scope), depth_lvl + 1);
+                print_indented!(self, format!("match_span: {:?}", match_span), depth_lvl + 1);
+
+                print_indented!(self, "arms: [", depth_lvl + 1);
+                for arm_id in arms.iter() {
+                    self.print_arm(*arm_id, depth_lvl + 2);
+                }
+                print_indented!(self, "]", depth_lvl + 1);
+                print_indented!(self, "}", depth_lvl);
+            }
             Let { expr, pat } => {
                 print_indented!(self, "Let {", depth_lvl);
                 print_indented!(self, "expr:", depth_lvl + 1);
@@ -415,6 +429,13 @@ impl<'a, 'tcx> ThirPrinter<'a, 'tcx> {
                 print_indented!(self, format!("label: {:?}", label), depth_lvl + 1);
                 print_indented!(self, "}", depth_lvl);
             }
+            ConstContinue { label, value } => {
+                print_indented!(self, "ConstContinue (", depth_lvl);
+                print_indented!(self, format!("label: {:?}", label), depth_lvl + 1);
+                print_indented!(self, "value:", depth_lvl + 1);
+                self.print_expr(*value, depth_lvl + 2);
+                print_indented!(self, ")", depth_lvl);
+            }
             Return { value } => {
                 print_indented!(self, "Return {", depth_lvl);
                 print_indented!(self, "value:", depth_lvl + 1);
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index 93c76c47f06..4e312aab497 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -847,7 +847,7 @@ impl<'a> Parser<'a> {
         self.dcx().emit_err(errors::LifetimeInBorrowExpression { span, lifetime_span: lt_span });
     }
 
-    /// Parse `mut?` or `raw [ const | mut ]`.
+    /// Parse `mut?` or `[ raw | pin ] [ const | mut ]`.
     fn parse_borrow_modifiers(&mut self) -> (ast::BorrowKind, ast::Mutability) {
         if self.check_keyword(exp!(Raw)) && self.look_ahead(1, Token::is_mutability) {
             // `raw [ const | mut ]`.
@@ -855,6 +855,11 @@ impl<'a> Parser<'a> {
             assert!(found_raw);
             let mutability = self.parse_const_or_mut().unwrap();
             (ast::BorrowKind::Raw, mutability)
+        } else if let Some((ast::Pinnedness::Pinned, mutbl)) = self.parse_pin_and_mut() {
+            // `pin [ const | mut ]`.
+            // `pin` has been gated in `self.parse_pin_and_mut()` so we don't
+            // need to gate it here.
+            (ast::BorrowKind::Pin, mutbl)
         } else {
             // `mut?`
             (ast::BorrowKind::Ref, self.parse_mutability())
diff --git a/compiler/rustc_parse/src/validate_attr.rs b/compiler/rustc_parse/src/validate_attr.rs
index 47e29923ee5..68d78af5943 100644
--- a/compiler/rustc_parse/src/validate_attr.rs
+++ b/compiler/rustc_parse/src/validate_attr.rs
@@ -301,6 +301,7 @@ fn emit_malformed_attribute(
             | sym::naked
             | sym::no_mangle
             | sym::must_use
+            | sym::track_caller
     ) {
         return;
     }
diff --git a/compiler/rustc_passes/messages.ftl b/compiler/rustc_passes/messages.ftl
index 29526817257..06acf7bf992 100644
--- a/compiler/rustc_passes/messages.ftl
+++ b/compiler/rustc_passes/messages.ftl
@@ -82,11 +82,14 @@ passes_collapse_debuginfo =
 passes_confusables = attribute should be applied to an inherent method
     .label = not an inherent method
 
+passes_const_continue_attr =
+    `#[const_continue]` should be applied to a break expression
+    .label = not a break expression
+
 passes_const_stable_not_stable =
     attribute `#[rustc_const_stable]` can only be applied to functions that are declared `#[stable]`
     .label = attribute specified here
 
-
 passes_coroutine_on_non_closure =
     attribute should be applied to closures
     .label = not a closure
@@ -446,6 +449,10 @@ passes_linkage =
     attribute should be applied to a function or static
     .label = not a function definition or static
 
+passes_loop_match_attr =
+    `#[loop_match]` should be applied to a loop
+    .label = not a loop
+
 passes_macro_export =
     `#[macro_export]` only has an effect on macro definitions
 
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index 25b84179443..99c220d946e 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -138,6 +138,12 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                 Attribute::Parsed(AttributeKind::Optimize(_, attr_span)) => {
                     self.check_optimize(hir_id, *attr_span, span, target)
                 }
+                Attribute::Parsed(AttributeKind::LoopMatch(attr_span)) => {
+                    self.check_loop_match(hir_id, *attr_span, target)
+                }
+                Attribute::Parsed(AttributeKind::ConstContinue(attr_span)) => {
+                    self.check_const_continue(hir_id, *attr_span, target)
+                }
                 Attribute::Parsed(AttributeKind::AllowInternalUnstable(syms)) => self
                     .check_allow_internal_unstable(
                         hir_id,
@@ -169,6 +175,9 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                 Attribute::Parsed(AttributeKind::Naked(attr_span)) => {
                     self.check_naked(hir_id, *attr_span, span, target)
                 }
+                Attribute::Parsed(AttributeKind::TrackCaller(attr_span)) => {
+                    self.check_track_caller(hir_id, *attr_span, attrs, span, target)
+                }
                 Attribute::Parsed(
                     AttributeKind::BodyStability { .. }
                     | AttributeKind::ConstStabilityIndirect
@@ -205,9 +214,6 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
                             self.check_target_feature(hir_id, attr, span, target, attrs)
                         }
                         [sym::thread_local, ..] => self.check_thread_local(attr, span, target),
-                        [sym::track_caller, ..] => {
-                            self.check_track_caller(hir_id, attr.span(), attrs, span, target)
-                        }
                         [sym::doc, ..] => self.check_doc_attrs(
                             attr,
                             attr_item.style,
@@ -722,9 +728,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
             // erroneously allowed it and some crates used it accidentally, to be compatible
             // with crates depending on them, we can't throw an error here.
             Target::Field | Target::Arm | Target::MacroDef => {
-                for attr in attrs {
-                    self.inline_attr_str_error_with_macro_def(hir_id, attr.span(), "track_caller");
-                }
+                self.inline_attr_str_error_with_macro_def(hir_id, attr_span, "track_caller");
             }
             _ => {
                 self.dcx().emit_err(errors::TrackedCallerWrongLocation {
@@ -2633,6 +2637,32 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
             }
         }
     }
+
+    fn check_loop_match(&self, hir_id: HirId, attr_span: Span, target: Target) {
+        let node_span = self.tcx.hir_span(hir_id);
+
+        if !matches!(target, Target::Expression) {
+            self.dcx().emit_err(errors::LoopMatchAttr { attr_span, node_span });
+            return;
+        }
+
+        if !matches!(self.tcx.hir_expect_expr(hir_id).kind, hir::ExprKind::Loop(..)) {
+            self.dcx().emit_err(errors::LoopMatchAttr { attr_span, node_span });
+        };
+    }
+
+    fn check_const_continue(&self, hir_id: HirId, attr_span: Span, target: Target) {
+        let node_span = self.tcx.hir_span(hir_id);
+
+        if !matches!(target, Target::Expression) {
+            self.dcx().emit_err(errors::ConstContinueAttr { attr_span, node_span });
+            return;
+        }
+
+        if !matches!(self.tcx.hir_expect_expr(hir_id).kind, hir::ExprKind::Break(..)) {
+            self.dcx().emit_err(errors::ConstContinueAttr { attr_span, node_span });
+        };
+    }
 }
 
 impl<'tcx> Visitor<'tcx> for CheckAttrVisitor<'tcx> {
diff --git a/compiler/rustc_passes/src/errors.rs b/compiler/rustc_passes/src/errors.rs
index 94c8ae77ed7..c933647165c 100644
--- a/compiler/rustc_passes/src/errors.rs
+++ b/compiler/rustc_passes/src/errors.rs
@@ -31,6 +31,24 @@ pub(crate) struct AutoDiffAttr {
     pub attr_span: Span,
 }
 
+#[derive(Diagnostic)]
+#[diag(passes_loop_match_attr)]
+pub(crate) struct LoopMatchAttr {
+    #[primary_span]
+    pub attr_span: Span,
+    #[label]
+    pub node_span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(passes_const_continue_attr)]
+pub(crate) struct ConstContinueAttr {
+    #[primary_span]
+    pub attr_span: Span,
+    #[label]
+    pub node_span: Span,
+}
+
 #[derive(LintDiagnostic)]
 #[diag(passes_outer_crate_level_attr)]
 pub(crate) struct OuterCrateLevelAttr;
diff --git a/compiler/rustc_passes/src/input_stats.rs b/compiler/rustc_passes/src/input_stats.rs
index 46e6c0bf7da..40bc18939d6 100644
--- a/compiler/rustc_passes/src/input_stats.rs
+++ b/compiler/rustc_passes/src/input_stats.rs
@@ -65,16 +65,16 @@ pub fn print_hir_stats(tcx: TyCtxt<'_>) {
         StatCollector { tcx: Some(tcx), nodes: FxHashMap::default(), seen: FxHashSet::default() };
     tcx.hir_walk_toplevel_module(&mut collector);
     tcx.hir_walk_attributes(&mut collector);
-    collector.print("HIR STATS", "hir-stats");
+    collector.print(tcx, "HIR STATS", "hir-stats");
 }
 
-pub fn print_ast_stats(krate: &ast::Crate, title: &str, prefix: &str) {
+pub fn print_ast_stats(tcx: TyCtxt<'_>, krate: &ast::Crate) {
     use rustc_ast::visit::Visitor;
 
     let mut collector =
         StatCollector { tcx: None, nodes: FxHashMap::default(), seen: FxHashSet::default() };
     collector.visit_crate(krate);
-    collector.print(title, prefix);
+    collector.print(tcx, "POST EXPANSION AST STATS", "ast-stats");
 }
 
 impl<'k> StatCollector<'k> {
@@ -116,29 +116,48 @@ impl<'k> StatCollector<'k> {
         }
     }
 
-    fn print(&self, title: &str, prefix: &str) {
+    fn print(&self, tcx: TyCtxt<'_>, title: &str, prefix: &str) {
+        use std::fmt::Write;
+
         // We will soon sort, so the initial order does not matter.
         #[allow(rustc::potential_query_instability)]
         let mut nodes: Vec<_> = self.nodes.iter().collect();
         nodes.sort_by_cached_key(|(label, node)| (node.stats.accum_size(), label.to_owned()));
+        nodes.reverse(); // bigger items first
+
+        let name_w = 18;
+        let acc_size1_w = 10;
+        let acc_size2_w = 8; // " (NN.N%)"
+        let acc_size_w = acc_size1_w + acc_size2_w;
+        let count_w = 14;
+        let item_size_w = 14;
+        let banner_w = name_w + acc_size_w + count_w + item_size_w;
 
         let total_size = nodes.iter().map(|(_, node)| node.stats.accum_size()).sum();
         let total_count = nodes.iter().map(|(_, node)| node.stats.count).sum();
 
-        eprintln!("{prefix} {title}");
-        eprintln!(
-            "{} {:<18}{:>18}{:>14}{:>14}",
-            prefix, "Name", "Accumulated Size", "Count", "Item Size"
+        // We write all the text into a string and print it with a single
+        // `eprint!`. This is an attempt to minimize interleaved text if multiple
+        // rustc processes are printing macro-stats at the same time (e.g. with
+        // `RUSTFLAGS='-Zinput-stats' cargo build`). It still doesn't guarantee
+        // non-interleaving, though.
+        let mut s = String::new();
+        _ = writeln!(s, "{prefix} {}", "=".repeat(banner_w));
+        _ = writeln!(s, "{prefix} {title}: {}", tcx.crate_name(hir::def_id::LOCAL_CRATE));
+        _ = writeln!(
+            s,
+            "{prefix} {:<name_w$}{:>acc_size_w$}{:>count_w$}{:>item_size_w$}",
+            "Name", "Accumulated Size", "Count", "Item Size"
         );
-        eprintln!("{prefix} ----------------------------------------------------------------");
+        _ = writeln!(s, "{prefix} {}", "-".repeat(banner_w));
 
         let percent = |m, n| (m * 100) as f64 / n as f64;
 
         for (label, node) in nodes {
             let size = node.stats.accum_size();
-            eprintln!(
-                "{} {:<18}{:>10} ({:4.1}%){:>14}{:>14}",
-                prefix,
+            _ = writeln!(
+                s,
+                "{prefix} {:<name_w$}{:>acc_size1_w$} ({:4.1}%){:>count_w$}{:>item_size_w$}",
                 label,
                 usize_with_underscores(size),
                 percent(size, total_size),
@@ -155,9 +174,9 @@ impl<'k> StatCollector<'k> {
 
                 for (label, subnode) in subnodes {
                     let size = subnode.accum_size();
-                    eprintln!(
-                        "{} - {:<18}{:>10} ({:4.1}%){:>14}",
-                        prefix,
+                    _ = writeln!(
+                        s,
+                        "{prefix} - {:<name_w$}{:>acc_size1_w$} ({:4.1}%){:>count_w$}",
                         label,
                         usize_with_underscores(size),
                         percent(size, total_size),
@@ -166,15 +185,17 @@ impl<'k> StatCollector<'k> {
                 }
             }
         }
-        eprintln!("{prefix} ----------------------------------------------------------------");
-        eprintln!(
-            "{} {:<18}{:>10}        {:>14}",
-            prefix,
+        _ = writeln!(s, "{prefix} {}", "-".repeat(banner_w));
+        _ = writeln!(
+            s,
+            "{prefix} {:<name_w$}{:>acc_size1_w$}{:>acc_size2_w$}{:>count_w$}",
             "Total",
             usize_with_underscores(total_size),
+            "",
             usize_with_underscores(total_count),
         );
-        eprintln!("{prefix}");
+        _ = writeln!(s, "{prefix} {}", "=".repeat(banner_w));
+        eprint!("{s}");
     }
 }
 
diff --git a/compiler/rustc_pattern_analysis/src/constructor.rs b/compiler/rustc_pattern_analysis/src/constructor.rs
index f7a4931c111..09685640e50 100644
--- a/compiler/rustc_pattern_analysis/src/constructor.rs
+++ b/compiler/rustc_pattern_analysis/src/constructor.rs
@@ -314,7 +314,8 @@ impl IntRange {
         IntRange { lo, hi }
     }
 
-    fn is_subrange(&self, other: &Self) -> bool {
+    #[inline]
+    pub fn is_subrange(&self, other: &Self) -> bool {
         other.lo <= self.lo && self.hi <= other.hi
     }
 
diff --git a/compiler/rustc_resolve/src/imports.rs b/compiler/rustc_resolve/src/imports.rs
index e989209e177..1b7a2c3bda0 100644
--- a/compiler/rustc_resolve/src/imports.rs
+++ b/compiler/rustc_resolve/src/imports.rs
@@ -608,7 +608,9 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
             }
         }
 
-        self.throw_unresolved_import_error(errors, glob_error);
+        if !errors.is_empty() {
+            self.throw_unresolved_import_error(errors, glob_error);
+        }
     }
 
     pub(crate) fn check_hidden_glob_reexports(
@@ -688,14 +690,19 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
             Some(def_id) if self.mods_with_parse_errors.contains(&def_id) => false,
             _ => true,
         });
+        errors.retain(|(_import, err)| {
+            // If we've encountered something like `use _;`, we've already emitted an error stating
+            // that `_` is not a valid identifier, so we ignore that resolve error.
+            err.segment != Some(kw::Underscore)
+        });
+
         if errors.is_empty() {
+            self.tcx.dcx().delayed_bug("expected a parse or \"`_` can't be an identifier\" error");
             return;
         }
 
-        /// Upper limit on the number of `span_label` messages.
-        const MAX_LABEL_COUNT: usize = 10;
-
         let span = MultiSpan::from_spans(errors.iter().map(|(_, err)| err.span).collect());
+
         let paths = errors
             .iter()
             .map(|(import, err)| {
@@ -715,6 +722,9 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
             diag.note(note.clone());
         }
 
+        /// Upper limit on the number of `span_label` messages.
+        const MAX_LABEL_COUNT: usize = 10;
+
         for (import, err) in errors.into_iter().take(MAX_LABEL_COUNT) {
             if let Some(label) = err.label {
                 diag.span_label(err.span, label);
diff --git a/compiler/rustc_session/Cargo.toml b/compiler/rustc_session/Cargo.toml
index f0ee19e3c67..5b88a7017c5 100644
--- a/compiler/rustc_session/Cargo.toml
+++ b/compiler/rustc_session/Cargo.toml
@@ -22,7 +22,6 @@ rustc_macros = { path = "../rustc_macros" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
-smallvec = "1.8.1"
 termize = "0.1.1"
 tracing = "0.1"
 # tidy-alphabetical-end
diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs
index 87e4b0a17aa..c62e4ac30ea 100644
--- a/compiler/rustc_session/src/config.rs
+++ b/compiler/rustc_session/src/config.rs
@@ -1296,6 +1296,28 @@ bitflags::bitflags! {
     }
 }
 
+#[derive(Clone, Debug)]
+pub struct Sysroot {
+    pub explicit: Option<PathBuf>,
+    pub default: PathBuf,
+}
+
+impl Sysroot {
+    pub fn new(explicit: Option<PathBuf>) -> Sysroot {
+        Sysroot { explicit, default: filesearch::default_sysroot() }
+    }
+
+    /// Return explicit sysroot if it was passed with `--sysroot`, or default sysroot otherwise.
+    pub fn path(&self) -> &Path {
+        self.explicit.as_deref().unwrap_or(&self.default)
+    }
+
+    /// Returns both explicit sysroot if it was passed with `--sysroot` and the default sysroot.
+    pub fn all_paths(&self) -> impl Iterator<Item = &Path> {
+        self.explicit.as_deref().into_iter().chain(iter::once(&*self.default))
+    }
+}
+
 pub fn host_tuple() -> &'static str {
     // Get the host triple out of the build environment. This ensures that our
     // idea of the host triple is the same as for the set of libraries we've
@@ -1342,7 +1364,7 @@ impl Default for Options {
             describe_lints: false,
             output_types: OutputTypes(BTreeMap::new()),
             search_paths: vec![],
-            sysroot: filesearch::materialize_sysroot(None),
+            sysroot: Sysroot::new(None),
             target_triple: TargetTuple::from_tuple(host_tuple()),
             test: false,
             incremental: None,
@@ -2673,7 +2695,6 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M
 
     let cg = cg;
 
-    let sysroot_opt = matches.opt_str("sysroot").map(|m| PathBuf::from(&m));
     let target_triple = parse_target_triple(early_dcx, matches);
     let opt_level = parse_opt_level(early_dcx, matches, &cg);
     // The `-g` and `-C debuginfo` flags specify the same setting, so we want to be able
@@ -2712,10 +2733,10 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M
 
     let logical_env = parse_logical_env(early_dcx, matches);
 
-    let sysroot = filesearch::materialize_sysroot(sysroot_opt);
+    let sysroot = Sysroot::new(matches.opt_str("sysroot").map(PathBuf::from));
 
     let real_source_base_dir = |suffix: &str, confirm: &str| {
-        let mut candidate = sysroot.join(suffix);
+        let mut candidate = sysroot.path().join(suffix);
         if let Ok(metadata) = candidate.symlink_metadata() {
             // Replace the symlink bootstrap creates, with its destination.
             // We could try to use `fs::canonicalize` instead, but that might
@@ -2742,7 +2763,7 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M
     let mut search_paths = vec![];
     for s in &matches.opt_strs("L") {
         search_paths.push(SearchPath::from_cli_opt(
-            &sysroot,
+            sysroot.path(),
             &target_triple,
             early_dcx,
             s,
diff --git a/compiler/rustc_session/src/filesearch.rs b/compiler/rustc_session/src/filesearch.rs
index 4f8c3926207..f64fa86948c 100644
--- a/compiler/rustc_session/src/filesearch.rs
+++ b/compiler/rustc_session/src/filesearch.rs
@@ -5,7 +5,6 @@ use std::{env, fs};
 
 use rustc_fs_util::try_canonicalize;
 use rustc_target::spec::Target;
-use smallvec::{SmallVec, smallvec};
 
 use crate::search_paths::{PathKind, SearchPath};
 
@@ -182,24 +181,9 @@ fn current_dll_path() -> Result<PathBuf, String> {
     Err("current_dll_path is not supported on WASI".to_string())
 }
 
-pub fn sysroot_with_fallback(sysroot: &Path) -> SmallVec<[PathBuf; 2]> {
-    let mut candidates = smallvec![sysroot.to_owned()];
-    let default_sysroot = get_or_default_sysroot();
-    if default_sysroot != sysroot {
-        candidates.push(default_sysroot);
-    }
-    candidates
-}
-
-/// Returns the provided sysroot or calls [`get_or_default_sysroot`] if it's none.
-/// Panics if [`get_or_default_sysroot`]  returns an error.
-pub fn materialize_sysroot(maybe_sysroot: Option<PathBuf>) -> PathBuf {
-    maybe_sysroot.unwrap_or_else(|| get_or_default_sysroot())
-}
-
 /// This function checks if sysroot is found using env::args().next(), and if it
 /// is not found, finds sysroot from current rustc_driver dll.
-pub fn get_or_default_sysroot() -> PathBuf {
+pub(crate) fn default_sysroot() -> PathBuf {
     fn default_from_rustc_driver_dll() -> Result<PathBuf, String> {
         let dll = current_dll_path()?;
 
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index 7fef942525b..232531dc673 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -337,7 +337,7 @@ top_level_options!(
         output_types: OutputTypes [TRACKED],
         search_paths: Vec<SearchPath> [UNTRACKED],
         libs: Vec<NativeLib> [TRACKED],
-        sysroot: PathBuf [UNTRACKED],
+        sysroot: Sysroot [UNTRACKED],
 
         target_triple: TargetTuple [TRACKED],
 
diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs
index ad58c3c8f7d..bad2581ae31 100644
--- a/compiler/rustc_session/src/session.rs
+++ b/compiler/rustc_session/src/session.rs
@@ -149,7 +149,6 @@ pub struct Session {
     pub opts: config::Options,
     pub target_tlib_path: Arc<SearchPath>,
     pub psess: ParseSess,
-    pub sysroot: PathBuf,
     /// Input, input file path and output file path to this compilation process.
     pub io: CompilerIO,
 
@@ -456,8 +455,10 @@ impl Session {
     /// directories are also returned, for example if `--sysroot` is used but tools are missing
     /// (#125246): we also add the bin directories to the sysroot where rustc is located.
     pub fn get_tools_search_paths(&self, self_contained: bool) -> Vec<PathBuf> {
-        let search_paths = filesearch::sysroot_with_fallback(&self.sysroot)
-            .into_iter()
+        let search_paths = self
+            .opts
+            .sysroot
+            .all_paths()
             .map(|sysroot| filesearch::make_target_bin_path(&sysroot, config::host_tuple()));
 
         if self_contained {
@@ -1028,7 +1029,6 @@ pub fn build_session(
     fluent_resources: Vec<&'static str>,
     driver_lint_caps: FxHashMap<lint::LintId, lint::Level>,
     target: Target,
-    sysroot: PathBuf,
     cfg_version: &'static str,
     ice_file: Option<PathBuf>,
     using_internal_features: &'static AtomicBool,
@@ -1063,7 +1063,7 @@ pub fn build_session(
     }
 
     let host_triple = TargetTuple::from_tuple(config::host_tuple());
-    let (host, target_warnings) = Target::search(&host_triple, &sysroot)
+    let (host, target_warnings) = Target::search(&host_triple, sopts.sysroot.path())
         .unwrap_or_else(|e| dcx.handle().fatal(format!("Error loading host specification: {e}")));
     for warning in target_warnings.warning_messages() {
         dcx.handle().warn(warning)
@@ -1096,13 +1096,14 @@ pub fn build_session(
     let host_triple = config::host_tuple();
     let target_triple = sopts.target_triple.tuple();
     // FIXME use host sysroot?
-    let host_tlib_path = Arc::new(SearchPath::from_sysroot_and_triple(&sysroot, host_triple));
+    let host_tlib_path =
+        Arc::new(SearchPath::from_sysroot_and_triple(sopts.sysroot.path(), host_triple));
     let target_tlib_path = if host_triple == target_triple {
         // Use the same `SearchPath` if host and target triple are identical to avoid unnecessary
         // rescanning of the target lib path and an unnecessary allocation.
         Arc::clone(&host_tlib_path)
     } else {
-        Arc::new(SearchPath::from_sysroot_and_triple(&sysroot, target_triple))
+        Arc::new(SearchPath::from_sysroot_and_triple(sopts.sysroot.path(), target_triple))
     };
 
     let prof = SelfProfilerRef::new(
@@ -1134,7 +1135,6 @@ pub fn build_session(
         opts: sopts,
         target_tlib_path,
         psess,
-        sysroot,
         io,
         incr_comp_session: RwLock::new(IncrCompSession::NotInitialized),
         prof,
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 5861ae4a372..c9262d24a17 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -693,6 +693,7 @@ symbols! {
         const_closures,
         const_compare_raw_pointers,
         const_constructor,
+        const_continue,
         const_deallocate,
         const_destruct,
         const_eval_limit,
@@ -1305,6 +1306,7 @@ symbols! {
         logf64,
         loongarch_target_feature,
         loop_break_value,
+        loop_match,
         lt,
         m68k_target_feature,
         macro_at_most_once_rep,
diff --git a/compiler/rustc_ty_utils/src/consts.rs b/compiler/rustc_ty_utils/src/consts.rs
index b275cd382ab..60f8bd9d83a 100644
--- a/compiler/rustc_ty_utils/src/consts.rs
+++ b/compiler/rustc_ty_utils/src/consts.rs
@@ -226,7 +226,11 @@ fn recurse_build<'tcx>(
         ExprKind::Yield { .. } => {
             error(GenericConstantTooComplexSub::YieldNotSupported(node.span))?
         }
-        ExprKind::Continue { .. } | ExprKind::Break { .. } | ExprKind::Loop { .. } => {
+        ExprKind::Continue { .. }
+        | ExprKind::ConstContinue { .. }
+        | ExprKind::Break { .. }
+        | ExprKind::Loop { .. }
+        | ExprKind::LoopMatch { .. } => {
             error(GenericConstantTooComplexSub::LoopNotSupported(node.span))?
         }
         ExprKind::Box { .. } => error(GenericConstantTooComplexSub::BoxNotSupported(node.span))?,
@@ -329,6 +333,7 @@ impl<'a, 'tcx> IsThirPolymorphic<'a, 'tcx> {
             | thir::ExprKind::NeverToAny { .. }
             | thir::ExprKind::PointerCoercion { .. }
             | thir::ExprKind::Loop { .. }
+            | thir::ExprKind::LoopMatch { .. }
             | thir::ExprKind::Let { .. }
             | thir::ExprKind::Match { .. }
             | thir::ExprKind::Block { .. }
@@ -342,6 +347,7 @@ impl<'a, 'tcx> IsThirPolymorphic<'a, 'tcx> {
             | thir::ExprKind::RawBorrow { .. }
             | thir::ExprKind::Break { .. }
             | thir::ExprKind::Continue { .. }
+            | thir::ExprKind::ConstContinue { .. }
             | thir::ExprKind::Return { .. }
             | thir::ExprKind::Become { .. }
             | thir::ExprKind::Array { .. }