about summary refs log tree commit diff
path: root/compiler/rustc_middle/src
diff options
context:
space:
mode:
Diffstat (limited to 'compiler/rustc_middle/src')
-rw-r--r--compiler/rustc_middle/src/error.rs17
-rw-r--r--compiler/rustc_middle/src/hir/map.rs26
-rw-r--r--compiler/rustc_middle/src/hir/mod.rs10
-rw-r--r--compiler/rustc_middle/src/lib.rs3
-rw-r--r--compiler/rustc_middle/src/middle/codegen_fn_attrs.rs3
-rw-r--r--compiler/rustc_middle/src/middle/exported_symbols.rs2
-rw-r--r--compiler/rustc_middle/src/mir/interpret/error.rs52
-rw-r--r--compiler/rustc_middle/src/mir/interpret/mod.rs4
-rw-r--r--compiler/rustc_middle/src/mir/interpret/queries.rs120
-rw-r--r--compiler/rustc_middle/src/query/erase.rs7
-rw-r--r--compiler/rustc_middle/src/query/mod.rs2
-rw-r--r--compiler/rustc_middle/src/thir.rs13
-rw-r--r--compiler/rustc_middle/src/thir/visit.rs3
-rw-r--r--compiler/rustc_middle/src/traits/mod.rs20
-rw-r--r--compiler/rustc_middle/src/ty/closure.rs82
-rw-r--r--compiler/rustc_middle/src/ty/consts.rs13
-rw-r--r--compiler/rustc_middle/src/ty/consts/valtree.rs8
-rw-r--r--compiler/rustc_middle/src/ty/context.rs89
-rw-r--r--compiler/rustc_middle/src/ty/context/tls.rs2
-rw-r--r--compiler/rustc_middle/src/ty/inhabitedness/mod.rs10
-rw-r--r--compiler/rustc_middle/src/ty/layout.rs7
-rw-r--r--compiler/rustc_middle/src/ty/mod.rs35
-rw-r--r--compiler/rustc_middle/src/ty/structural_impls.rs12
-rw-r--r--compiler/rustc_middle/src/ty/sty.rs65
-rw-r--r--compiler/rustc_middle/src/ty/util.rs1
25 files changed, 379 insertions, 227 deletions
diff --git a/compiler/rustc_middle/src/error.rs b/compiler/rustc_middle/src/error.rs
index 6c6b12fed67..f36ae831653 100644
--- a/compiler/rustc_middle/src/error.rs
+++ b/compiler/rustc_middle/src/error.rs
@@ -170,3 +170,20 @@ pub(crate) struct TypeLengthLimit {
     pub path: PathBuf,
     pub type_length: usize,
 }
+
+#[derive(Diagnostic)]
+#[diag(middle_max_num_nodes_in_valtree)]
+pub(crate) struct MaxNumNodesInValtree {
+    #[primary_span]
+    pub span: Span,
+    pub global_const_id: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(middle_invalid_const_in_valtree)]
+#[note]
+pub(crate) struct InvalidConstInValtree {
+    #[primary_span]
+    pub span: Span,
+    pub global_const_id: String,
+}
diff --git a/compiler/rustc_middle/src/hir/map.rs b/compiler/rustc_middle/src/hir/map.rs
index e5e1ae508ed..291707878a3 100644
--- a/compiler/rustc_middle/src/hir/map.rs
+++ b/compiler/rustc_middle/src/hir/map.rs
@@ -328,8 +328,7 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     /// Returns an iterator of the `DefId`s for all body-owners in this
-    /// crate. If you would prefer to iterate over the bodies
-    /// themselves, you can do `self.hir_crate(()).body_ids.iter()`.
+    /// crate.
     #[inline]
     pub fn hir_body_owners(self) -> impl Iterator<Item = LocalDefId> {
         self.hir_crate_items(()).body_owners.iter().copied()
@@ -396,12 +395,11 @@ impl<'tcx> TyCtxt<'tcx> {
     where
         V: Visitor<'tcx>,
     {
-        let krate = self.hir_crate(());
-        for info in krate.owners.iter() {
-            if let MaybeOwner::Owner(info) = info {
-                for attrs in info.attrs.map.values() {
-                    walk_list!(visitor, visit_attribute, *attrs);
-                }
+        let krate = self.hir_crate_items(());
+        for owner in krate.owners() {
+            let attrs = self.hir_attr_map(owner);
+            for attrs in attrs.map.values() {
+                walk_list!(visitor, visit_attribute, *attrs);
             }
         }
         V::Result::output()
@@ -1225,6 +1223,7 @@ pub(super) fn hir_module_items(tcx: TyCtxt<'_>, module_id: LocalModDefId) -> Mod
         ..
     } = collector;
     ModuleItems {
+        add_root: false,
         submodules: submodules.into_boxed_slice(),
         free_items: items.into_boxed_slice(),
         trait_items: trait_items.into_boxed_slice(),
@@ -1255,11 +1254,20 @@ pub(crate) fn hir_crate_items(tcx: TyCtxt<'_>, _: ()) -> ModuleItems {
         body_owners,
         opaques,
         nested_bodies,
-        delayed_lint_items,
+        mut delayed_lint_items,
         ..
     } = collector;
 
+    // The crate could have delayed lints too, but would not be picked up by the visitor.
+    // The `delayed_lint_items` list is smart - it only contains items which we know from
+    // earlier passes is guaranteed to contain lints. It's a little harder to determine that
+    // for sure here, so we simply always add the crate to the list. If it has no lints,
+    // we'll discover that later. The cost of this should be low, there's only one crate
+    // after all compared to the many items we have we wouldn't want to iterate over later.
+    delayed_lint_items.push(CRATE_OWNER_ID);
+
     ModuleItems {
+        add_root: true,
         submodules: submodules.into_boxed_slice(),
         free_items: items.into_boxed_slice(),
         trait_items: trait_items.into_boxed_slice(),
diff --git a/compiler/rustc_middle/src/hir/mod.rs b/compiler/rustc_middle/src/hir/mod.rs
index 9f79ed4b5a5..d7a8dce0536 100644
--- a/compiler/rustc_middle/src/hir/mod.rs
+++ b/compiler/rustc_middle/src/hir/mod.rs
@@ -24,6 +24,9 @@ use crate::ty::{EarlyBinder, ImplSubject, TyCtxt};
 /// bodies. The Ids are in visitor order. This is used to partition a pass between modules.
 #[derive(Debug, HashStable, Encodable, Decodable)]
 pub struct ModuleItems {
+    /// Whether this represents the whole crate, in which case we need to add `CRATE_OWNER_ID` to
+    /// the iterators if we want to account for the crate root.
+    add_root: bool,
     submodules: Box<[OwnerId]>,
     free_items: Box<[ItemId]>,
     trait_items: Box<[TraitItemId]>,
@@ -66,9 +69,10 @@ impl ModuleItems {
     }
 
     pub fn owners(&self) -> impl Iterator<Item = OwnerId> {
-        self.free_items
-            .iter()
-            .map(|id| id.owner_id)
+        self.add_root
+            .then_some(CRATE_OWNER_ID)
+            .into_iter()
+            .chain(self.free_items.iter().map(|id| id.owner_id))
             .chain(self.trait_items.iter().map(|id| id.owner_id))
             .chain(self.impl_items.iter().map(|id| id.owner_id))
             .chain(self.foreign_items.iter().map(|id| id.owner_id))
diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs
index 6cb1d8c5fc4..ce2cb33c173 100644
--- a/compiler/rustc_middle/src/lib.rs
+++ b/compiler/rustc_middle/src/lib.rs
@@ -28,6 +28,7 @@
 #![allow(internal_features)]
 #![allow(rustc::diagnostic_outside_of_impl)]
 #![allow(rustc::untranslatable_diagnostic)]
+#![cfg_attr(not(bootstrap), allow(rustc::direct_use_of_rustc_type_ir))]
 #![cfg_attr(not(bootstrap), feature(sized_hierarchy))]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]
@@ -39,12 +40,12 @@
 #![feature(box_patterns)]
 #![feature(closure_track_caller)]
 #![feature(core_intrinsics)]
-#![feature(coroutines)]
 #![feature(debug_closure_helpers)]
 #![feature(decl_macro)]
 #![feature(discriminant_kind)]
 #![feature(extern_types)]
 #![feature(file_buffered)]
+#![feature(gen_blocks)]
 #![feature(if_let_guard)]
 #![feature(intra_doc_pointers)]
 #![feature(iter_from_coroutine)]
diff --git a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
index f21cf5fa45e..2f16d385efb 100644
--- a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
+++ b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
@@ -47,8 +47,7 @@ pub struct CodegenFnAttrs {
     /// be generated against a specific instruction set. Only usable on architectures which allow
     /// switching between multiple instruction sets.
     pub instruction_set: Option<InstructionSetAttr>,
-    /// The `#[repr(align(...))]` attribute. Indicates the value of which the function should be
-    /// aligned to.
+    /// The `#[align(...)]` attribute. Determines the alignment of the function body.
     pub alignment: Option<Align>,
     /// The `#[patchable_function_entry(...)]` attribute. Indicates how many nops should be around
     /// the function entry.
diff --git a/compiler/rustc_middle/src/middle/exported_symbols.rs b/compiler/rustc_middle/src/middle/exported_symbols.rs
index 1d67d0fe3bb..64a1f2aff15 100644
--- a/compiler/rustc_middle/src/middle/exported_symbols.rs
+++ b/compiler/rustc_middle/src/middle/exported_symbols.rs
@@ -22,7 +22,7 @@ impl SymbolExportLevel {
 }
 
 /// Kind of exported symbols.
-#[derive(Eq, PartialEq, Debug, Copy, Clone, Encodable, Decodable, HashStable)]
+#[derive(Eq, PartialEq, Debug, Copy, Clone, Encodable, Decodable, HashStable, Hash)]
 pub enum SymbolExportKind {
     Text,
     Data,
diff --git a/compiler/rustc_middle/src/mir/interpret/error.rs b/compiler/rustc_middle/src/mir/interpret/error.rs
index 6ff3cac049b..41a166083d0 100644
--- a/compiler/rustc_middle/src/mir/interpret/error.rs
+++ b/compiler/rustc_middle/src/mir/interpret/error.rs
@@ -35,7 +35,7 @@ impl From<ReportedErrorInfo> for ErrorHandled {
 }
 
 impl ErrorHandled {
-    pub fn with_span(self, span: Span) -> Self {
+    pub(crate) fn with_span(self, span: Span) -> Self {
         match self {
             ErrorHandled::Reported(err, _span) => ErrorHandled::Reported(err, span),
             ErrorHandled::TooGeneric(_span) => ErrorHandled::TooGeneric(span),
@@ -94,14 +94,51 @@ impl From<ReportedErrorInfo> for ErrorGuaranteed {
     }
 }
 
+/// An error type for the `const_to_valtree` query. Some error should be reported with a "use-site span",
+/// which means the query cannot emit the error, so those errors are represented as dedicated variants here.
+#[derive(Debug, Copy, Clone, PartialEq, Eq, HashStable, TyEncodable, TyDecodable)]
+pub enum ValTreeCreationError<'tcx> {
+    /// The constant is too big to be valtree'd.
+    NodesOverflow,
+    /// The constant references mutable or external memory, so it cannot be valtree'd.
+    InvalidConst,
+    /// Values of this type, or this particular value, are not supported as valtrees.
+    NonSupportedType(Ty<'tcx>),
+    /// The error has already been handled by const evaluation.
+    ErrorHandled(ErrorHandled),
+}
+
+impl<'tcx> From<ErrorHandled> for ValTreeCreationError<'tcx> {
+    fn from(err: ErrorHandled) -> Self {
+        ValTreeCreationError::ErrorHandled(err)
+    }
+}
+
+impl<'tcx> From<InterpErrorInfo<'tcx>> for ValTreeCreationError<'tcx> {
+    fn from(err: InterpErrorInfo<'tcx>) -> Self {
+        // An error ocurred outside the const-eval query, as part of constructing the valtree. We
+        // don't currently preserve the details of this error, since `InterpErrorInfo` cannot be put
+        // into a query result and it can only be access of some mutable or external memory.
+        let (_kind, backtrace) = err.into_parts();
+        backtrace.print_backtrace();
+        ValTreeCreationError::InvalidConst
+    }
+}
+
+impl<'tcx> ValTreeCreationError<'tcx> {
+    pub(crate) fn with_span(self, span: Span) -> Self {
+        use ValTreeCreationError::*;
+        match self {
+            ErrorHandled(handled) => ErrorHandled(handled.with_span(span)),
+            other => other,
+        }
+    }
+}
+
 pub type EvalToAllocationRawResult<'tcx> = Result<ConstAlloc<'tcx>, ErrorHandled>;
 pub type EvalStaticInitializerRawResult<'tcx> = Result<ConstAllocation<'tcx>, ErrorHandled>;
 pub type EvalToConstValueResult<'tcx> = Result<ConstValue<'tcx>, ErrorHandled>;
-/// `Ok(Err(ty))` indicates the constant was fine, but the valtree couldn't be constructed
-/// because the value contains something of type `ty` that is not valtree-compatible.
-/// The caller can then show an appropriate error; the query does not have the
-/// necessary context to give good user-facing errors for this case.
-pub type EvalToValTreeResult<'tcx> = Result<Result<ValTree<'tcx>, Ty<'tcx>>, ErrorHandled>;
+pub type EvalToValTreeResult<'tcx> = Result<ValTree<'tcx>, ValTreeCreationError<'tcx>>;
 
 #[cfg(target_pointer_width = "64")]
 rustc_data_structures::static_assert_size!(InterpErrorInfo<'_>, 8);
@@ -450,10 +487,9 @@ pub enum ValidationErrorKind<'tcx> {
         ptr_kind: PointerKind,
         ty: Ty<'tcx>,
     },
-    ConstRefToMutable,
-    ConstRefToExtern,
     MutableRefToImmutable,
     UnsafeCellInImmutable,
+    MutableRefInConst,
     NullFnPtr,
     NeverVal,
     NullablePtrOutOfRange {
diff --git a/compiler/rustc_middle/src/mir/interpret/mod.rs b/compiler/rustc_middle/src/mir/interpret/mod.rs
index c2438af6a1e..ea2f84d46d7 100644
--- a/compiler/rustc_middle/src/mir/interpret/mod.rs
+++ b/compiler/rustc_middle/src/mir/interpret/mod.rs
@@ -38,8 +38,8 @@ pub use self::error::{
     EvalToAllocationRawResult, EvalToConstValueResult, EvalToValTreeResult, ExpectedKind,
     InterpErrorInfo, InterpErrorKind, InterpResult, InvalidMetaKind, InvalidProgramInfo,
     MachineStopType, Misalignment, PointerKind, ReportedErrorInfo, ResourceExhaustionInfo,
-    ScalarSizeMismatch, UndefinedBehaviorInfo, UnsupportedOpInfo, ValidationErrorInfo,
-    ValidationErrorKind, interp_ok,
+    ScalarSizeMismatch, UndefinedBehaviorInfo, UnsupportedOpInfo, ValTreeCreationError,
+    ValidationErrorInfo, ValidationErrorKind, interp_ok,
 };
 pub use self::pointer::{CtfeProvenance, Pointer, PointerArithmetic, Provenance};
 pub use self::value::Scalar;
diff --git a/compiler/rustc_middle/src/mir/interpret/queries.rs b/compiler/rustc_middle/src/mir/interpret/queries.rs
index 97db45a70d7..e25f35c59c2 100644
--- a/compiler/rustc_middle/src/mir/interpret/queries.rs
+++ b/compiler/rustc_middle/src/mir/interpret/queries.rs
@@ -5,11 +5,11 @@ use rustc_span::{DUMMY_SP, Span};
 use tracing::{debug, instrument};
 
 use super::{
-    ErrorHandled, EvalToAllocationRawResult, EvalToConstValueResult, EvalToValTreeResult, GlobalId,
-    ReportedErrorInfo,
+    ErrorHandled, EvalToAllocationRawResult, EvalToConstValueResult, GlobalId, ReportedErrorInfo,
 };
-use crate::mir;
-use crate::ty::{self, GenericArgs, TyCtxt, TypeVisitableExt};
+use crate::mir::interpret::ValTreeCreationError;
+use crate::ty::{self, ConstToValTreeResult, GenericArgs, TyCtxt, TypeVisitableExt};
+use crate::{error, mir};
 
 impl<'tcx> TyCtxt<'tcx> {
     /// Evaluates a constant without providing any generic parameters. This is useful to evaluate consts
@@ -92,7 +92,7 @@ impl<'tcx> TyCtxt<'tcx> {
         typing_env: ty::TypingEnv<'tcx>,
         ct: ty::UnevaluatedConst<'tcx>,
         span: Span,
-    ) -> EvalToValTreeResult<'tcx> {
+    ) -> ConstToValTreeResult<'tcx> {
         // Cannot resolve `Unevaluated` constants that contain inference
         // variables. We reject those here since `resolve`
         // would fail otherwise.
@@ -103,47 +103,54 @@ impl<'tcx> TyCtxt<'tcx> {
             bug!("did not expect inference variables here");
         }
 
-        match ty::Instance::try_resolve(self, typing_env, ct.def, ct.args) {
-            Ok(Some(instance)) => {
-                let cid = GlobalId { instance, promoted: None };
-                self.const_eval_global_id_for_typeck(typing_env, cid, span).inspect(|_| {
-                    // We are emitting the lint here instead of in `is_const_evaluatable`
-                    // as we normalize obligations before checking them, and normalization
-                    // uses this function to evaluate this constant.
-                    //
-                    // @lcnr believes that successfully evaluating even though there are
-                    // used generic parameters is a bug of evaluation, so checking for it
-                    // here does feel somewhat sensible.
-                    if !self.features().generic_const_exprs()
-                        && ct.args.has_non_region_param()
-                        // We only FCW for anon consts as repeat expr counts with anon consts are the only place
-                        // that we have a back compat hack for. We don't need to check this is a const argument
-                        // as only anon consts as const args should get evaluated "for the type system".
-                        //
-                        // If we don't *only* FCW anon consts we can wind up incorrectly FCW'ing uses of assoc
-                        // consts in pattern positions. #140447
-                        && self.def_kind(instance.def_id()) == DefKind::AnonConst
-                    {
-                        let mir_body = self.mir_for_ctfe(instance.def_id());
-                        if mir_body.is_polymorphic {
-                            let Some(local_def_id) = ct.def.as_local() else { return };
-                            self.node_span_lint(
-                                lint::builtin::CONST_EVALUATABLE_UNCHECKED,
-                                self.local_def_id_to_hir_id(local_def_id),
-                                self.def_span(ct.def),
-                                |lint| { lint.primary_message("cannot use constants which depend on generic parameters in types"); },
-                            )
-                        }
-                    }
-                })
-            }
+        let cid = match ty::Instance::try_resolve(self, typing_env, ct.def, ct.args) {
+            Ok(Some(instance)) => GlobalId { instance, promoted: None },
             // For errors during resolution, we deliberately do not point at the usage site of the constant,
             // since for these errors the place the constant is used shouldn't matter.
-            Ok(None) => Err(ErrorHandled::TooGeneric(DUMMY_SP)),
+            Ok(None) => return Err(ErrorHandled::TooGeneric(DUMMY_SP).into()),
             Err(err) => {
-                Err(ErrorHandled::Reported(ReportedErrorInfo::non_const_eval_error(err), DUMMY_SP))
+                return Err(ErrorHandled::Reported(
+                    ReportedErrorInfo::non_const_eval_error(err),
+                    DUMMY_SP,
+                )
+                .into());
             }
-        }
+        };
+
+        self.const_eval_global_id_for_typeck(typing_env, cid, span).inspect(|_| {
+            // We are emitting the lint here instead of in `is_const_evaluatable`
+            // as we normalize obligations before checking them, and normalization
+            // uses this function to evaluate this constant.
+            //
+            // @lcnr believes that successfully evaluating even though there are
+            // used generic parameters is a bug of evaluation, so checking for it
+            // here does feel somewhat sensible.
+            if !self.features().generic_const_exprs()
+                && ct.args.has_non_region_param()
+                // We only FCW for anon consts as repeat expr counts with anon consts are the only place
+                // that we have a back compat hack for. We don't need to check this is a const argument
+                // as only anon consts as const args should get evaluated "for the type system".
+                //
+                // If we don't *only* FCW anon consts we can wind up incorrectly FCW'ing uses of assoc
+                // consts in pattern positions. #140447
+                && self.def_kind(cid.instance.def_id()) == DefKind::AnonConst
+            {
+                let mir_body = self.mir_for_ctfe(cid.instance.def_id());
+                if mir_body.is_polymorphic {
+                    let Some(local_def_id) = ct.def.as_local() else { return };
+                    self.node_span_lint(
+                        lint::builtin::CONST_EVALUATABLE_UNCHECKED,
+                        self.local_def_id_to_hir_id(local_def_id),
+                        self.def_span(ct.def),
+                        |lint| {
+                            lint.primary_message(
+                                "cannot use constants which depend on generic parameters in types",
+                            );
+                        },
+                    )
+                }
+            }
+        })
     }
 
     pub fn const_eval_instance(
@@ -182,17 +189,42 @@ impl<'tcx> TyCtxt<'tcx> {
         typing_env: ty::TypingEnv<'tcx>,
         cid: GlobalId<'tcx>,
         span: Span,
-    ) -> EvalToValTreeResult<'tcx> {
+    ) -> ConstToValTreeResult<'tcx> {
         // Const-eval shouldn't depend on lifetimes at all, so we can erase them, which should
         // improve caching of queries.
         let inputs =
             self.erase_regions(typing_env.with_post_analysis_normalized(self).as_query_input(cid));
         debug!(?inputs);
-        if !span.is_dummy() {
+        let res = if !span.is_dummy() {
             // The query doesn't know where it is being invoked, so we need to fix the span.
             self.at(span).eval_to_valtree(inputs).map_err(|e| e.with_span(span))
         } else {
             self.eval_to_valtree(inputs)
+        };
+        match res {
+            Ok(valtree) => Ok(Ok(valtree)),
+            Err(err) => {
+                match err {
+                    // Let the caller decide how to handle this.
+                    ValTreeCreationError::NonSupportedType(ty) => Ok(Err(ty)),
+                    // Report the others.
+                    ValTreeCreationError::NodesOverflow => {
+                        let handled = self.dcx().emit_err(error::MaxNumNodesInValtree {
+                            span,
+                            global_const_id: cid.display(self),
+                        });
+                        Err(ReportedErrorInfo::allowed_in_infallible(handled).into())
+                    }
+                    ValTreeCreationError::InvalidConst => {
+                        let handled = self.dcx().emit_err(error::InvalidConstInValtree {
+                            span,
+                            global_const_id: cid.display(self),
+                        });
+                        Err(ReportedErrorInfo::allowed_in_infallible(handled).into())
+                    }
+                    ValTreeCreationError::ErrorHandled(handled) => Err(handled),
+                }
+            }
         }
     }
 }
diff --git a/compiler/rustc_middle/src/query/erase.rs b/compiler/rustc_middle/src/query/erase.rs
index 7c998761a9d..26a31cb055e 100644
--- a/compiler/rustc_middle/src/query/erase.rs
+++ b/compiler/rustc_middle/src/query/erase.rs
@@ -4,6 +4,7 @@ use std::mem::MaybeUninit;
 
 use rustc_span::ErrorGuaranteed;
 
+use crate::mir::interpret::EvalToValTreeResult;
 use crate::query::CyclePlaceholder;
 use crate::ty::adjustment::CoerceUnsizedInfo;
 use crate::ty::{self, Ty, TyCtxt};
@@ -156,10 +157,8 @@ impl EraseType for Result<mir::ConstValue<'_>, mir::interpret::ErrorHandled> {
     type Result = [u8; size_of::<Result<mir::ConstValue<'static>, mir::interpret::ErrorHandled>>()];
 }
 
-impl EraseType for Result<Result<ty::ValTree<'_>, Ty<'_>>, mir::interpret::ErrorHandled> {
-    type Result = [u8; size_of::<
-        Result<Result<ty::ValTree<'static>, Ty<'static>>, mir::interpret::ErrorHandled>,
-    >()];
+impl EraseType for EvalToValTreeResult<'_> {
+    type Result = [u8; size_of::<EvalToValTreeResult<'static>>()];
 }
 
 impl EraseType for Result<&'_ ty::List<Ty<'_>>, ty::util::AlwaysRequiresDrop> {
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
index 930d9fba433..3668f4e12f5 100644
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@@ -1576,7 +1576,7 @@ rustc_queries! {
     query vtable_allocation(key: (Ty<'tcx>, Option<ty::ExistentialTraitRef<'tcx>>)) -> mir::interpret::AllocId {
         desc { |tcx| "vtable const allocation for <{} as {}>",
             key.0,
-            key.1.map(|trait_ref| format!("{trait_ref}")).unwrap_or("_".to_owned())
+            key.1.map(|trait_ref| format!("{trait_ref}")).unwrap_or_else(|| "_".to_owned())
         }
     }
 
diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs
index b9a014d14c0..d0e72a86d8a 100644
--- a/compiler/rustc_middle/src/thir.rs
+++ b/compiler/rustc_middle/src/thir.rs
@@ -378,6 +378,14 @@ pub enum ExprKind<'tcx> {
     Loop {
         body: ExprId,
     },
+    /// A `#[loop_match] loop { state = 'blk: { match state { ... } } }` expression.
+    LoopMatch {
+        /// The state variable that is updated, and also the scrutinee of the match.
+        state: ExprId,
+        region_scope: region::Scope,
+        arms: Box<[ArmId]>,
+        match_span: Span,
+    },
     /// Special expression representing the `let` part of an `if let` or similar construct
     /// (including `if let` guards in match arms, and let-chains formed by `&&`).
     ///
@@ -454,6 +462,11 @@ pub enum ExprKind<'tcx> {
     Continue {
         label: region::Scope,
     },
+    /// A `#[const_continue] break` expression.
+    ConstContinue {
+        label: region::Scope,
+        value: ExprId,
+    },
     /// A `return` expression.
     Return {
         value: Option<ExprId>,
diff --git a/compiler/rustc_middle/src/thir/visit.rs b/compiler/rustc_middle/src/thir/visit.rs
index d8743814d79..c9ef723aea4 100644
--- a/compiler/rustc_middle/src/thir/visit.rs
+++ b/compiler/rustc_middle/src/thir/visit.rs
@@ -83,7 +83,7 @@ pub fn walk_expr<'thir, 'tcx: 'thir, V: Visitor<'thir, 'tcx>>(
             visitor.visit_pat(pat);
         }
         Loop { body } => visitor.visit_expr(&visitor.thir()[body]),
-        Match { scrutinee, ref arms, .. } => {
+        LoopMatch { state: scrutinee, ref arms, .. } | Match { scrutinee, ref arms, .. } => {
             visitor.visit_expr(&visitor.thir()[scrutinee]);
             for &arm in &**arms {
                 visitor.visit_arm(&visitor.thir()[arm]);
@@ -108,6 +108,7 @@ pub fn walk_expr<'thir, 'tcx: 'thir, V: Visitor<'thir, 'tcx>>(
             }
         }
         Continue { label: _ } => {}
+        ConstContinue { value, label: _ } => visitor.visit_expr(&visitor.thir()[value]),
         Return { value } => {
             if let Some(value) = value {
                 visitor.visit_expr(&visitor.thir()[value])
diff --git a/compiler/rustc_middle/src/traits/mod.rs b/compiler/rustc_middle/src/traits/mod.rs
index 27079af06fc..1a5a9765ce7 100644
--- a/compiler/rustc_middle/src/traits/mod.rs
+++ b/compiler/rustc_middle/src/traits/mod.rs
@@ -332,7 +332,11 @@ pub enum ObligationCauseCode<'tcx> {
     },
 
     /// Computing common supertype in an if expression
-    IfExpression(Box<IfExpressionCause<'tcx>>),
+    IfExpression {
+        expr_id: HirId,
+        // Is the expectation of this match expression an RPIT?
+        tail_defines_return_position_impl_trait: Option<LocalDefId>,
+    },
 
     /// Computing common supertype of an if expression with no else counter-part
     IfExpressionWithNoElse,
@@ -397,6 +401,8 @@ pub enum ObligationCauseCode<'tcx> {
 
     RustCall,
 
+    DynCompatible(Span),
+
     /// Obligations to prove that a `Drop` or negative auto trait impl is not stronger than
     /// the ADT it's being implemented for.
     AlwaysApplicableImpl,
@@ -548,18 +554,6 @@ pub struct PatternOriginExpr {
     pub peeled_prefix_suggestion_parentheses: bool,
 }
 
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-#[derive(TypeFoldable, TypeVisitable, HashStable, TyEncodable, TyDecodable)]
-pub struct IfExpressionCause<'tcx> {
-    pub then_id: HirId,
-    pub else_id: HirId,
-    pub then_ty: Ty<'tcx>,
-    pub else_ty: Ty<'tcx>,
-    pub outer_span: Option<Span>,
-    // Is the expectation of this match expression an RPIT?
-    pub tail_defines_return_position_impl_trait: Option<LocalDefId>,
-}
-
 #[derive(Clone, Debug, PartialEq, Eq, HashStable, TyEncodable, TyDecodable)]
 #[derive(TypeVisitable, TypeFoldable)]
 pub struct DerivedCause<'tcx> {
diff --git a/compiler/rustc_middle/src/ty/closure.rs b/compiler/rustc_middle/src/ty/closure.rs
index df67bb505a6..b8c7d6cf3b1 100644
--- a/compiler/rustc_middle/src/ty/closure.rs
+++ b/compiler/rustc_middle/src/ty/closure.rs
@@ -422,53 +422,49 @@ pub fn analyze_coroutine_closure_captures<'a, 'tcx: 'a, T>(
     child_captures: impl IntoIterator<Item = &'a CapturedPlace<'tcx>>,
     mut for_each: impl FnMut((usize, &'a CapturedPlace<'tcx>), (usize, &'a CapturedPlace<'tcx>)) -> T,
 ) -> impl Iterator<Item = T> {
-    std::iter::from_coroutine(
-        #[coroutine]
-        move || {
-            let mut child_captures = child_captures.into_iter().enumerate().peekable();
-
-            // One parent capture may correspond to several child captures if we end up
-            // refining the set of captures via edition-2021 precise captures. We want to
-            // match up any number of child captures with one parent capture, so we keep
-            // peeking off this `Peekable` until the child doesn't match anymore.
-            for (parent_field_idx, parent_capture) in parent_captures.into_iter().enumerate() {
-                // Make sure we use every field at least once, b/c why are we capturing something
-                // if it's not used in the inner coroutine.
-                let mut field_used_at_least_once = false;
-
-                // A parent matches a child if they share the same prefix of projections.
-                // The child may have more, if it is capturing sub-fields out of
-                // something that is captured by-move in the parent closure.
-                while child_captures.peek().is_some_and(|(_, child_capture)| {
-                    child_prefix_matches_parent_projections(parent_capture, child_capture)
-                }) {
-                    let (child_field_idx, child_capture) = child_captures.next().unwrap();
-                    // This analysis only makes sense if the parent capture is a
-                    // prefix of the child capture.
-                    assert!(
-                        child_capture.place.projections.len()
-                            >= parent_capture.place.projections.len(),
-                        "parent capture ({parent_capture:#?}) expected to be prefix of \
+    gen move {
+        let mut child_captures = child_captures.into_iter().enumerate().peekable();
+
+        // One parent capture may correspond to several child captures if we end up
+        // refining the set of captures via edition-2021 precise captures. We want to
+        // match up any number of child captures with one parent capture, so we keep
+        // peeking off this `Peekable` until the child doesn't match anymore.
+        for (parent_field_idx, parent_capture) in parent_captures.into_iter().enumerate() {
+            // Make sure we use every field at least once, b/c why are we capturing something
+            // if it's not used in the inner coroutine.
+            let mut field_used_at_least_once = false;
+
+            // A parent matches a child if they share the same prefix of projections.
+            // The child may have more, if it is capturing sub-fields out of
+            // something that is captured by-move in the parent closure.
+            while child_captures.peek().is_some_and(|(_, child_capture)| {
+                child_prefix_matches_parent_projections(parent_capture, child_capture)
+            }) {
+                let (child_field_idx, child_capture) = child_captures.next().unwrap();
+                // This analysis only makes sense if the parent capture is a
+                // prefix of the child capture.
+                assert!(
+                    child_capture.place.projections.len() >= parent_capture.place.projections.len(),
+                    "parent capture ({parent_capture:#?}) expected to be prefix of \
                     child capture ({child_capture:#?})"
-                    );
-
-                    yield for_each(
-                        (parent_field_idx, parent_capture),
-                        (child_field_idx, child_capture),
-                    );
-
-                    field_used_at_least_once = true;
-                }
+                );
 
-                // Make sure the field was used at least once.
-                assert!(
-                    field_used_at_least_once,
-                    "we captured {parent_capture:#?} but it was not used in the child coroutine?"
+                yield for_each(
+                    (parent_field_idx, parent_capture),
+                    (child_field_idx, child_capture),
                 );
+
+                field_used_at_least_once = true;
             }
-            assert_eq!(child_captures.next(), None, "leftover child captures?");
-        },
-    )
+
+            // Make sure the field was used at least once.
+            assert!(
+                field_used_at_least_once,
+                "we captured {parent_capture:#?} but it was not used in the child coroutine?"
+            );
+        }
+        assert_eq!(child_captures.next(), None, "leftover child captures?");
+    }
 }
 
 fn child_prefix_matches_parent_projections(
diff --git a/compiler/rustc_middle/src/ty/consts.rs b/compiler/rustc_middle/src/ty/consts.rs
index 2fbaa2221a1..fd1aa4042bc 100644
--- a/compiler/rustc_middle/src/ty/consts.rs
+++ b/compiler/rustc_middle/src/ty/consts.rs
@@ -144,6 +144,19 @@ impl<'tcx> Const<'tcx> {
         let reported = tcx.dcx().span_delayed_bug(span, msg);
         Const::new_error(tcx, reported)
     }
+
+    pub fn is_trivially_wf(self) -> bool {
+        match self.kind() {
+            ty::ConstKind::Param(_) | ty::ConstKind::Placeholder(_) | ty::ConstKind::Bound(..) => {
+                true
+            }
+            ty::ConstKind::Infer(_)
+            | ty::ConstKind::Unevaluated(..)
+            | ty::ConstKind::Value(_)
+            | ty::ConstKind::Error(_)
+            | ty::ConstKind::Expr(_) => false,
+        }
+    }
 }
 
 impl<'tcx> rustc_type_ir::inherent::Const<TyCtxt<'tcx>> for Const<'tcx> {
diff --git a/compiler/rustc_middle/src/ty/consts/valtree.rs b/compiler/rustc_middle/src/ty/consts/valtree.rs
index 2f21d19e03c..d95006dcf4a 100644
--- a/compiler/rustc_middle/src/ty/consts/valtree.rs
+++ b/compiler/rustc_middle/src/ty/consts/valtree.rs
@@ -5,7 +5,7 @@ use rustc_data_structures::intern::Interned;
 use rustc_macros::{HashStable, Lift, TyDecodable, TyEncodable, TypeFoldable, TypeVisitable};
 
 use super::ScalarInt;
-use crate::mir::interpret::Scalar;
+use crate::mir::interpret::{ErrorHandled, Scalar};
 use crate::ty::{self, Ty, TyCtxt};
 
 /// This datastructure is used to represent the value of constants used in the type system.
@@ -124,6 +124,12 @@ impl fmt::Debug for ValTree<'_> {
     }
 }
 
+/// `Ok(Err(ty))` indicates the constant was fine, but the valtree couldn't be constructed
+/// because the value contains something of type `ty` that is not valtree-compatible.
+/// The caller can then show an appropriate error; the query does not have the
+/// necessary context to give good user-facing errors for this case.
+pub type ConstToValTreeResult<'tcx> = Result<Result<ValTree<'tcx>, Ty<'tcx>>, ErrorHandled>;
+
 /// A type-level constant value.
 ///
 /// Represents a typed, fully evaluated constant.
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index f1395c242f2..457a4f4d502 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -432,6 +432,13 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
         self.explicit_implied_predicates_of(def_id).map_bound(|preds| preds.into_iter().copied())
     }
 
+    fn impl_super_outlives(
+        self,
+        impl_def_id: DefId,
+    ) -> ty::EarlyBinder<'tcx, impl IntoIterator<Item = ty::Clause<'tcx>>> {
+        self.impl_super_outlives(impl_def_id)
+    }
+
     fn impl_is_const(self, def_id: DefId) -> bool {
         debug_assert_matches!(self.def_kind(def_id), DefKind::Impl { of_trait: true });
         self.is_conditionally_const(def_id)
@@ -691,14 +698,6 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
         self.unsizing_params_for_adt(adt_def_id)
     }
 
-    fn find_const_ty_from_env(
-        self,
-        param_env: ty::ParamEnv<'tcx>,
-        placeholder: Self::PlaceholderConst,
-    ) -> Ty<'tcx> {
-        placeholder.find_const_ty_from_env(param_env)
-    }
-
     fn anonymize_bound_vars<T: TypeFoldable<TyCtxt<'tcx>>>(
         self,
         binder: ty::Binder<'tcx, T>,
@@ -774,8 +773,8 @@ bidirectional_lang_item_map! {
     Future,
     FutureOutput,
     Iterator,
-    Metadata,
     MetaSized,
+    Metadata,
     Option,
     PointeeSized,
     PointeeTrait,
@@ -1479,6 +1478,12 @@ pub struct GlobalCtxt<'tcx> {
 
     pub canonical_param_env_cache: CanonicalParamEnvCache<'tcx>,
 
+    /// Caches the index of the highest bound var in clauses in a canonical binder.
+    pub highest_var_in_clauses_cache: Lock<FxHashMap<ty::Clauses<'tcx>, usize>>,
+    /// Caches the instantiation of a canonical binder given a set of args.
+    pub clauses_cache:
+        Lock<FxHashMap<(ty::Clauses<'tcx>, &'tcx [ty::GenericArg<'tcx>]), ty::Clauses<'tcx>>>,
+
     /// Data layout specification for the current target.
     pub data_layout: TargetDataLayout,
 
@@ -1727,6 +1732,8 @@ impl<'tcx> TyCtxt<'tcx> {
             new_solver_evaluation_cache: Default::default(),
             new_solver_canonical_param_env_cache: Default::default(),
             canonical_param_env_cache: Default::default(),
+            highest_var_in_clauses_cache: Default::default(),
+            clauses_cache: Default::default(),
             data_layout,
             alloc_map: interpret::AllocMap::new(),
             current_gcx,
@@ -2079,23 +2086,20 @@ impl<'tcx> TyCtxt<'tcx> {
         self.dep_graph.read_index(DepNodeIndex::FOREVER_RED_NODE);
 
         let definitions = &self.untracked.definitions;
-        std::iter::from_coroutine(
-            #[coroutine]
-            || {
-                let mut i = 0;
-
-                // Recompute the number of definitions each time, because our caller may be creating
-                // new ones.
-                while i < { definitions.read().num_definitions() } {
-                    let local_def_index = rustc_span::def_id::DefIndex::from_usize(i);
-                    yield LocalDefId { local_def_index };
-                    i += 1;
-                }
+        gen {
+            let mut i = 0;
+
+            // Recompute the number of definitions each time, because our caller may be creating
+            // new ones.
+            while i < { definitions.read().num_definitions() } {
+                let local_def_index = rustc_span::def_id::DefIndex::from_usize(i);
+                yield LocalDefId { local_def_index };
+                i += 1;
+            }
 
-                // Freeze definitions once we finish iterating on them, to prevent adding new ones.
-                definitions.freeze();
-            },
-        )
+            // Freeze definitions once we finish iterating on them, to prevent adding new ones.
+            definitions.freeze();
+        }
     }
 
     pub fn def_path_table(self) -> &'tcx rustc_hir::definitions::DefPathTable {
@@ -2113,7 +2117,7 @@ impl<'tcx> TyCtxt<'tcx> {
     ) -> &'tcx rustc_hir::def_path_hash_map::DefPathHashMap {
         // Create a dependency to the crate to be sure we re-execute this when the amount of
         // definitions change.
-        self.ensure_ok().hir_crate(());
+        self.ensure_ok().hir_crate_items(());
         // Freeze definitions once we start iterating on them, to prevent adding new ones
         // while iterating. If some query needs to add definitions, it should be `ensure`d above.
         self.untracked.definitions.freeze().def_path_hash_to_def_index_map()
@@ -3160,42 +3164,33 @@ impl<'tcx> TyCtxt<'tcx> {
         lint_level(self.sess, lint, level, Some(span.into()), decorate);
     }
 
-    /// Find the crate root and the appropriate span where `use` and outer attributes can be
-    /// inserted at.
-    pub fn crate_level_attribute_injection_span(self, hir_id: HirId) -> Option<Span> {
-        for (_hir_id, node) in self.hir_parent_iter(hir_id) {
-            if let hir::Node::Crate(m) = node {
-                return Some(m.spans.inject_use_span.shrink_to_lo());
-            }
-        }
-        None
+    /// Find the appropriate span where `use` and outer attributes can be inserted at.
+    pub fn crate_level_attribute_injection_span(self) -> Span {
+        let node = self.hir_node(hir::CRATE_HIR_ID);
+        let hir::Node::Crate(m) = node else { bug!() };
+        m.spans.inject_use_span.shrink_to_lo()
     }
 
     pub fn disabled_nightly_features<E: rustc_errors::EmissionGuarantee>(
         self,
         diag: &mut Diag<'_, E>,
-        hir_id: Option<HirId>,
         features: impl IntoIterator<Item = (String, Symbol)>,
     ) {
         if !self.sess.is_nightly_build() {
             return;
         }
 
-        let span = hir_id.and_then(|id| self.crate_level_attribute_injection_span(id));
+        let span = self.crate_level_attribute_injection_span();
         for (desc, feature) in features {
             // FIXME: make this string translatable
             let msg =
                 format!("add `#![feature({feature})]` to the crate attributes to enable{desc}");
-            if let Some(span) = span {
-                diag.span_suggestion_verbose(
-                    span,
-                    msg,
-                    format!("#![feature({feature})]\n"),
-                    Applicability::MaybeIncorrect,
-                );
-            } else {
-                diag.help(msg);
-            }
+            diag.span_suggestion_verbose(
+                span,
+                msg,
+                format!("#![feature({feature})]\n"),
+                Applicability::MaybeIncorrect,
+            );
         }
     }
 
diff --git a/compiler/rustc_middle/src/ty/context/tls.rs b/compiler/rustc_middle/src/ty/context/tls.rs
index 5fc80bc7936..fa9995898ac 100644
--- a/compiler/rustc_middle/src/ty/context/tls.rs
+++ b/compiler/rustc_middle/src/ty/context/tls.rs
@@ -36,7 +36,7 @@ impl<'a, 'tcx> ImplicitCtxt<'a, 'tcx> {
 }
 
 // Import the thread-local variable from Rayon, which is preserved for Rayon jobs.
-use rayon_core::tlv::TLV;
+use rustc_thread_pool::tlv::TLV;
 
 #[inline]
 fn erase(context: &ImplicitCtxt<'_, '_>) -> *const () {
diff --git a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
index d8bab58545f..2a336cc21f4 100644
--- a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
+++ b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
@@ -127,7 +127,9 @@ impl<'tcx> Ty<'tcx> {
                 InhabitedPredicate::True
             }
             Never => InhabitedPredicate::False,
-            Param(_) | Alias(ty::Projection | ty::Free, _) => InhabitedPredicate::GenericType(self),
+            Param(_) | Alias(ty::Inherent | ty::Projection | ty::Free, _) => {
+                InhabitedPredicate::GenericType(self)
+            }
             Alias(ty::Opaque, alias_ty) => {
                 match alias_ty.def_id.as_local() {
                     // Foreign opaque is considered inhabited.
@@ -139,12 +141,6 @@ impl<'tcx> Ty<'tcx> {
                     }
                 }
             }
-            // FIXME(inherent_associated_types): Most likely we can just map to `GenericType` like above.
-            // However it's unclear if the args passed to `InhabitedPredicate::instantiate` are of the correct
-            // format, i.e. don't contain parent args. If you hit this case, please verify this beforehand.
-            Alias(ty::Inherent, _) => {
-                bug!("unimplemented: inhabitedness checking for inherent projections")
-            }
             Tuple(tys) if tys.is_empty() => InhabitedPredicate::True,
             // use a query for more complex cases
             Adt(..) | Array(..) | Tuple(_) => tcx.inhabited_predicate_type(self),
diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs
index 13c281a6182..90b832df281 100644
--- a/compiler/rustc_middle/src/ty/layout.rs
+++ b/compiler/rustc_middle/src/ty/layout.rs
@@ -1248,11 +1248,12 @@ pub fn fn_can_unwind(tcx: TyCtxt<'_>, fn_def_id: Option<DefId>, abi: ExternAbi)
         | EfiApi
         | AvrInterrupt
         | AvrNonBlockingInterrupt
+        | CmseNonSecureCall
+        | CmseNonSecureEntry
+        | Custom
         | RiscvInterruptM
         | RiscvInterruptS
-        | CCmseNonSecureCall
-        | CCmseNonSecureEntry
-        | Custom
+        | RustInvalid
         | Unadjusted => false,
         Rust | RustCall | RustCold => tcx.sess.panic_strategy() == PanicStrategy::Unwind,
     }
diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs
index 97408e31854..425f5188cdb 100644
--- a/compiler/rustc_middle/src/ty/mod.rs
+++ b/compiler/rustc_middle/src/ty/mod.rs
@@ -75,8 +75,8 @@ pub use self::closure::{
     place_to_string_for_capture,
 };
 pub use self::consts::{
-    AnonConstKind, AtomicOrdering, Const, ConstInt, ConstKind, Expr, ExprKind, ScalarInt,
-    UnevaluatedConst, ValTree, ValTreeKind, Value,
+    AnonConstKind, AtomicOrdering, Const, ConstInt, ConstKind, ConstToValTreeResult, Expr,
+    ExprKind, ScalarInt, UnevaluatedConst, ValTree, ValTreeKind, Value,
 };
 pub use self::context::{
     CtxtInterners, CurrentGcx, DeducedParamAttrs, Feed, FreeRegionInfo, GlobalCtxt, Lift, TyCtxt,
@@ -652,6 +652,13 @@ impl<'tcx> Term<'tcx> {
         }
     }
 
+    pub fn is_trivially_wf(&self, tcx: TyCtxt<'tcx>) -> bool {
+        match self.kind() {
+            TermKind::Ty(ty) => ty.is_trivially_wf(tcx),
+            TermKind::Const(ct) => ct.is_trivially_wf(),
+        }
+    }
+
     /// Iterator that walks `self` and any types reachable from
     /// `self`, in depth-first order. Note that just walks the types
     /// that appear in `self`, it does not descend into the fields of
@@ -907,30 +914,6 @@ pub struct Placeholder<T> {
     pub universe: UniverseIndex,
     pub bound: T,
 }
-impl Placeholder<BoundVar> {
-    pub fn find_const_ty_from_env<'tcx>(self, env: ParamEnv<'tcx>) -> Ty<'tcx> {
-        let mut candidates = env.caller_bounds().iter().filter_map(|clause| {
-            // `ConstArgHasType` are never desugared to be higher ranked.
-            match clause.kind().skip_binder() {
-                ty::ClauseKind::ConstArgHasType(placeholder_ct, ty) => {
-                    assert!(!(placeholder_ct, ty).has_escaping_bound_vars());
-
-                    match placeholder_ct.kind() {
-                        ty::ConstKind::Placeholder(placeholder_ct) if placeholder_ct == self => {
-                            Some(ty)
-                        }
-                        _ => None,
-                    }
-                }
-                _ => None,
-            }
-        });
-
-        let ty = candidates.next().unwrap();
-        assert!(candidates.next().is_none());
-        ty
-    }
-}
 
 pub type PlaceholderRegion = Placeholder<BoundRegion>;
 
diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs
index 000ba7b6fa7..1214731a3b2 100644
--- a/compiler/rustc_middle/src/ty/structural_impls.rs
+++ b/compiler/rustc_middle/src/ty/structural_impls.rs
@@ -230,9 +230,9 @@ TrivialLiftImpls! {
     usize,
     u64,
     // tidy-alphabetical-start
+    crate::mir::Promoted,
     crate::mir::interpret::AllocId,
     crate::mir::interpret::Scalar,
-    crate::mir::Promoted,
     rustc_abi::ExternAbi,
     rustc_abi::Size,
     rustc_hir::Safety,
@@ -267,9 +267,6 @@ TrivialTypeTraversalImpls! {
     crate::mir::SwitchTargets,
     crate::traits::IsConstable,
     crate::traits::OverflowError,
-    crate::ty::abstract_const::NotConstEvaluatable,
-    crate::ty::adjustment::AutoBorrowMutability,
-    crate::ty::adjustment::PointerCoercion,
     crate::ty::AdtKind,
     crate::ty::AssocItem,
     crate::ty::AssocKind,
@@ -281,15 +278,18 @@ TrivialTypeTraversalImpls! {
     crate::ty::Placeholder<ty::BoundVar>,
     crate::ty::UserTypeAnnotationIndex,
     crate::ty::ValTree<'tcx>,
+    crate::ty::abstract_const::NotConstEvaluatable,
+    crate::ty::adjustment::AutoBorrowMutability,
+    crate::ty::adjustment::PointerCoercion,
     rustc_abi::FieldIdx,
     rustc_abi::VariantIdx,
     rustc_ast::InlineAsmOptions,
     rustc_ast::InlineAsmTemplatePiece,
     rustc_hir::CoroutineKind,
-    rustc_hir::def_id::LocalDefId,
     rustc_hir::HirId,
     rustc_hir::MatchSource,
     rustc_hir::RangeEnd,
+    rustc_hir::def_id::LocalDefId,
     rustc_span::Ident,
     rustc_span::Span,
     rustc_span::Symbol,
@@ -303,9 +303,9 @@ TrivialTypeTraversalImpls! {
 // interners).
 TrivialTypeTraversalAndLiftImpls! {
     // tidy-alphabetical-start
-    crate::ty::instance::ReifyReason,
     crate::ty::ParamConst,
     crate::ty::ParamTy,
+    crate::ty::instance::ReifyReason,
     rustc_hir::def_id::DefId,
     // tidy-alphabetical-end
 }
diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs
index 58829f72a72..3971ac13bbe 100644
--- a/compiler/rustc_middle/src/ty/sty.rs
+++ b/compiler/rustc_middle/src/ty/sty.rs
@@ -346,7 +346,7 @@ impl ParamConst {
     }
 
     #[instrument(level = "debug")]
-    pub fn find_ty_from_env<'tcx>(self, env: ParamEnv<'tcx>) -> Ty<'tcx> {
+    pub fn find_const_ty_from_env<'tcx>(self, env: ParamEnv<'tcx>) -> Ty<'tcx> {
         let mut candidates = env.caller_bounds().iter().filter_map(|clause| {
             // `ConstArgHasType` are never desugared to be higher ranked.
             match clause.kind().skip_binder() {
@@ -362,8 +362,19 @@ impl ParamConst {
             }
         });
 
-        let ty = candidates.next().unwrap();
-        assert!(candidates.next().is_none());
+        // N.B. it may be tempting to fix ICEs by making this function return
+        // `Option<Ty<'tcx>>` instead of `Ty<'tcx>`; however, this is generally
+        // considered to be a bandaid solution, since it hides more important
+        // underlying issues with how we construct generics and predicates of
+        // items. It's advised to fix the underlying issue rather than trying
+        // to modify this function.
+        let ty = candidates.next().unwrap_or_else(|| {
+            bug!("cannot find `{self:?}` in param-env: {env:#?}");
+        });
+        assert!(
+            candidates.next().is_none(),
+            "did not expect duplicate `ConstParamHasTy` for `{self:?}` in param-env: {env:#?}"
+        );
         ty
     }
 }
@@ -1832,7 +1843,7 @@ impl<'tcx> Ty<'tcx> {
             ty::Infer(ty::TyVar(_)) => false,
 
             ty::Infer(ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => {
-                bug!("`is_trivially_sized` applied to unexpected type: {:?}", self)
+                bug!("`has_trivial_sizedness` applied to unexpected type: {:?}", self)
             }
         }
     }
@@ -1896,6 +1907,52 @@ impl<'tcx> Ty<'tcx> {
         }
     }
 
+    pub fn is_trivially_wf(self, tcx: TyCtxt<'tcx>) -> bool {
+        match *self.kind() {
+            ty::Bool
+            | ty::Char
+            | ty::Int(_)
+            | ty::Uint(_)
+            | ty::Float(_)
+            | ty::Str
+            | ty::Never
+            | ty::Param(_)
+            | ty::Placeholder(_)
+            | ty::Bound(..) => true,
+
+            ty::Slice(ty) => {
+                ty.is_trivially_wf(tcx) && ty.has_trivial_sizedness(tcx, SizedTraitKind::Sized)
+            }
+            ty::RawPtr(ty, _) => ty.is_trivially_wf(tcx),
+
+            ty::FnPtr(sig_tys, _) => {
+                sig_tys.skip_binder().inputs_and_output.iter().all(|ty| ty.is_trivially_wf(tcx))
+            }
+            ty::Ref(_, ty, _) => ty.is_global() && ty.is_trivially_wf(tcx),
+
+            ty::Infer(infer) => match infer {
+                ty::TyVar(_) => false,
+                ty::IntVar(_) | ty::FloatVar(_) => true,
+                ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_) => true,
+            },
+
+            ty::Adt(_, _)
+            | ty::Tuple(_)
+            | ty::Array(..)
+            | ty::Foreign(_)
+            | ty::Pat(_, _)
+            | ty::FnDef(..)
+            | ty::UnsafeBinder(..)
+            | ty::Dynamic(..)
+            | ty::Closure(..)
+            | ty::CoroutineClosure(..)
+            | ty::Coroutine(..)
+            | ty::CoroutineWitness(..)
+            | ty::Alias(..)
+            | ty::Error(_) => false,
+        }
+    }
+
     /// If `self` is a primitive, return its [`Symbol`].
     pub fn primitive_symbol(self) -> Option<Symbol> {
         match self.kind() {
diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index 51f57e71ce9..69b8be3d9cb 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -768,6 +768,7 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn def_kind_descr(self, def_kind: DefKind, def_id: DefId) -> &'static str {
         match def_kind {
             DefKind::AssocFn if self.associated_item(def_id).is_method() => "method",
+            DefKind::AssocTy if self.opt_rpitit_info(def_id).is_some() => "opaque type",
             DefKind::Closure if let Some(coroutine_kind) = self.coroutine_kind(def_id) => {
                 match coroutine_kind {
                     hir::CoroutineKind::Desugared(