Auto merge of #146765 - Zalathar:rollup-ewh4s9o, r=Zalathar

Rollup of 10 pull requests

Successful merges:

 - rust-lang/rust#146229 (Automatically switch to lto-fat when flag RUSTFLAGS="- Zautodiff=Enable" is set)
 - rust-lang/rust#146484 (rustdoc-search: JavaScript optimization based on Firefox Profiler output)
 - rust-lang/rust#146541 (std: simplify host lookup)
 - rust-lang/rust#146615 (rustc_codegen_llvm: Feature Conversion Tidying)
 - rust-lang/rust#146638 (`rustc_next_trait_solver`: canonical out of `EvalCtxt`)
 - rust-lang/rust#146663 (Allow windows resource compiler to be overridden)
 - rust-lang/rust#146691 (std: Fix WASI implementation of `remove_dir_all`)
 - rust-lang/rust#146709 (stdarch subtree update)
 - rust-lang/rust#146738 (Fix tidy spellchecking on Windows)
 - rust-lang/rust#146740 (miri subtree update)

r? `@ghost`
`@rustbot` modify labels: rollup

16 files changed, 613 insertions, 586 deletions

diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs
index 8461c8b03d5..45c5c9aa551 100644
--- a/compiler/rustc_codegen_llvm/src/llvm_util.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs
@@ -217,27 +217,16 @@ impl<'a> IntoIterator for LLVMFeature<'a> {
 /// Rust can also be build with an external precompiled version of LLVM which might lead to failures
 /// if the oldest tested / supported LLVM version doesn't yet support the relevant intrinsics.
 pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFeature<'a>> {
-    let arch = if sess.target.arch == "x86_64" {
-        "x86"
-    } else if sess.target.arch == "arm64ec" {
-        "aarch64"
-    } else if sess.target.arch == "sparc64" {
-        "sparc"
-    } else if sess.target.arch == "powerpc64" {
-        "powerpc"
-    } else {
-        &*sess.target.arch
+    let raw_arch = &*sess.target.arch;
+    let arch = match raw_arch {
+        "x86_64" => "x86",
+        "arm64ec" => "aarch64",
+        "sparc64" => "sparc",
+        "powerpc64" => "powerpc",
+        _ => raw_arch,
     };
+    let (major, _, _) = get_version();
     match (arch, s) {
-        ("x86", "sse4.2") => Some(LLVMFeature::with_dependencies(
-            "sse4.2",
-            smallvec![TargetFeatureFoldStrength::EnableOnly("crc32")],
-        )),
-        ("x86", "pclmulqdq") => Some(LLVMFeature::new("pclmul")),
-        ("x86", "rdrand") => Some(LLVMFeature::new("rdrnd")),
-        ("x86", "bmi1") => Some(LLVMFeature::new("bmi")),
-        ("x86", "cmpxchg16b") => Some(LLVMFeature::new("cx16")),
-        ("x86", "lahfsahf") => Some(LLVMFeature::new("sahf")),
         ("aarch64", "rcpc2") => Some(LLVMFeature::new("rcpc-immo")),
         ("aarch64", "dpb") => Some(LLVMFeature::new("ccpp")),
         ("aarch64", "dpb2") => Some(LLVMFeature::new("ccdp")),
@@ -260,14 +249,23 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea
         ("aarch64", "fpmr") => None, // only existed in 18
         ("arm", "fp16") => Some(LLVMFeature::new("fullfp16")),
         // Filter out features that are not supported by the current LLVM version
-        ("loongarch32" | "loongarch64", "32s") if get_version().0 < 21 => None,
+        ("loongarch32" | "loongarch64", "32s") if major < 21 => None,
+        ("powerpc", "power8-crypto") => Some(LLVMFeature::new("crypto")),
+        ("sparc", "leoncasa") => Some(LLVMFeature::new("hasleoncasa")),
+        ("x86", "sse4.2") => Some(LLVMFeature::with_dependencies(
+            "sse4.2",
+            smallvec![TargetFeatureFoldStrength::EnableOnly("crc32")],
+        )),
+        ("x86", "pclmulqdq") => Some(LLVMFeature::new("pclmul")),
+        ("x86", "rdrand") => Some(LLVMFeature::new("rdrnd")),
+        ("x86", "bmi1") => Some(LLVMFeature::new("bmi")),
+        ("x86", "cmpxchg16b") => Some(LLVMFeature::new("cx16")),
+        ("x86", "lahfsahf") => Some(LLVMFeature::new("sahf")),
         // Enable the evex512 target feature if an avx512 target feature is enabled.
         ("x86", s) if s.starts_with("avx512") => Some(LLVMFeature::with_dependencies(
             s,
             smallvec![TargetFeatureFoldStrength::EnableOnly("evex512")],
         )),
-        ("sparc", "leoncasa") => Some(LLVMFeature::new("hasleoncasa")),
-        ("powerpc", "power8-crypto") => Some(LLVMFeature::new("crypto")),
         ("x86", "avx10.1") => Some(LLVMFeature::new("avx10.1-512")),
         ("x86", "avx10.2") => Some(LLVMFeature::new("avx10.2-512")),
         ("x86", "apxf") => Some(LLVMFeature::with_dependencies(
diff --git a/compiler/rustc_codegen_ssa/messages.ftl b/compiler/rustc_codegen_ssa/messages.ftl
index 1dd65d38a2b..91c3806df4c 100644
--- a/compiler/rustc_codegen_ssa/messages.ftl
+++ b/compiler/rustc_codegen_ssa/messages.ftl
@@ -8,8 +8,6 @@ codegen_ssa_aix_strip_not_used = using host's `strip` binary to cross-compile to
 
 codegen_ssa_archive_build_failure = failed to build archive at `{$path}`: {$error}
 
-codegen_ssa_autodiff_without_lto = using the autodiff feature requires using fat-lto
-
 codegen_ssa_bare_instruction_set = `#[instruction_set]` requires an argument
 
 codegen_ssa_binary_output_to_tty = option `-o` or `--emit` is used to write binary output type `{$shorthand}` to stdout, but stdout is a tty
diff --git a/compiler/rustc_codegen_ssa/src/errors.rs b/compiler/rustc_codegen_ssa/src/errors.rs
index fb5a8205140..d5c30c5c7a6 100644
--- a/compiler/rustc_codegen_ssa/src/errors.rs
+++ b/compiler/rustc_codegen_ssa/src/errors.rs
@@ -38,10 +38,6 @@ pub(crate) struct CguNotRecorded<'a> {
 }
 
 #[derive(Diagnostic)]
-#[diag(codegen_ssa_autodiff_without_lto)]
-pub struct AutodiffWithoutLto;
-
-#[derive(Diagnostic)]
 #[diag(codegen_ssa_unknown_reuse_kind)]
 pub(crate) struct UnknownReuseKind {
     #[primary_span]
diff --git a/compiler/rustc_next_trait_solver/src/canonicalizer.rs b/compiler/rustc_next_trait_solver/src/canonical/canonicalizer.rs
index 4b4ec4956eb..b25671d676b 100644
--- a/compiler/rustc_next_trait_solver/src/canonicalizer.rs
+++ b/compiler/rustc_next_trait_solver/src/canonical/canonicalizer.rs
@@ -57,7 +57,7 @@ enum CanonicalizeMode {
     },
 }
 
-pub struct Canonicalizer<'a, D: SolverDelegate<Interner = I>, I: Interner> {
+pub(super) struct Canonicalizer<'a, D: SolverDelegate<Interner = I>, I: Interner> {
     delegate: &'a D,
 
     // Immutable field.
@@ -83,7 +83,7 @@ pub struct Canonicalizer<'a, D: SolverDelegate<Interner = I>, I: Interner> {
 }
 
 impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
-    pub fn canonicalize_response<T: TypeFoldable<I>>(
+    pub(super) fn canonicalize_response<T: TypeFoldable<I>>(
         delegate: &'a D,
         max_input_universe: ty::UniverseIndex,
         variables: &'a mut Vec<I::GenericArg>,
@@ -112,7 +112,6 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
         let (max_universe, variables) = canonicalizer.finalize();
         Canonical { max_universe, variables, value }
     }
-
     fn canonicalize_param_env(
         delegate: &'a D,
         variables: &'a mut Vec<I::GenericArg>,
@@ -195,7 +194,7 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
     ///
     /// We want to keep the option of canonicalizing `'static` to an existential
     /// variable in the future by changing the way we detect global where-bounds.
-    pub fn canonicalize_input<P: TypeFoldable<I>>(
+    pub(super) fn canonicalize_input<P: TypeFoldable<I>>(
         delegate: &'a D,
         variables: &'a mut Vec<I::GenericArg>,
         input: QueryInput<I, P>,
diff --git a/compiler/rustc_next_trait_solver/src/canonical/mod.rs b/compiler/rustc_next_trait_solver/src/canonical/mod.rs
new file mode 100644
index 00000000000..e3520e238ed
--- /dev/null
+++ b/compiler/rustc_next_trait_solver/src/canonical/mod.rs
@@ -0,0 +1,364 @@
+//! Canonicalization is used to separate some goal from its context,
+//! throwing away unnecessary information in the process.
+//!
+//! This is necessary to cache goals containing inference variables
+//! and placeholders without restricting them to the current `InferCtxt`.
+//!
+//! Canonicalization is fairly involved, for more details see the relevant
+//! section of the [rustc-dev-guide][c].
+//!
+//! [c]: https://rustc-dev-guide.rust-lang.org/solve/canonicalization.html
+
+use std::iter;
+
+use canonicalizer::Canonicalizer;
+use rustc_index::IndexVec;
+use rustc_type_ir::inherent::*;
+use rustc_type_ir::relate::solver_relating::RelateExt;
+use rustc_type_ir::{
+    self as ty, Canonical, CanonicalVarKind, CanonicalVarValues, InferCtxtLike, Interner,
+    TypeFoldable,
+};
+use tracing::instrument;
+
+use crate::delegate::SolverDelegate;
+use crate::resolve::eager_resolve_vars;
+use crate::solve::{
+    CanonicalInput, CanonicalResponse, Certainty, ExternalConstraintsData, Goal,
+    NestedNormalizationGoals, PredefinedOpaquesData, QueryInput, Response, inspect,
+};
+
+pub mod canonicalizer;
+
+trait ResponseT<I: Interner> {
+    fn var_values(&self) -> CanonicalVarValues<I>;
+}
+
+impl<I: Interner> ResponseT<I> for Response<I> {
+    fn var_values(&self) -> CanonicalVarValues<I> {
+        self.var_values
+    }
+}
+
+impl<I: Interner, T> ResponseT<I> for inspect::State<I, T> {
+    fn var_values(&self) -> CanonicalVarValues<I> {
+        self.var_values
+    }
+}
+
+/// Canonicalizes the goal remembering the original values
+/// for each bound variable.
+///
+/// This expects `goal` and `opaque_types` to be eager resolved.
+pub(super) fn canonicalize_goal<D, I>(
+    delegate: &D,
+    goal: Goal<I, I::Predicate>,
+    opaque_types: Vec<(ty::OpaqueTypeKey<I>, I::Ty)>,
+) -> (Vec<I::GenericArg>, CanonicalInput<I, I::Predicate>)
+where
+    D: SolverDelegate<Interner = I>,
+    I: Interner,
+{
+    let mut orig_values = Default::default();
+    let canonical = Canonicalizer::canonicalize_input(
+        delegate,
+        &mut orig_values,
+        QueryInput {
+            goal,
+            predefined_opaques_in_body: delegate
+                .cx()
+                .mk_predefined_opaques_in_body(PredefinedOpaquesData { opaque_types }),
+        },
+    );
+    let query_input = ty::CanonicalQueryInput { canonical, typing_mode: delegate.typing_mode() };
+    (orig_values, query_input)
+}
+
+pub(super) fn canonicalize_response<D, I, T>(
+    delegate: &D,
+    max_input_universe: ty::UniverseIndex,
+    value: T,
+) -> ty::Canonical<I, T>
+where
+    D: SolverDelegate<Interner = I>,
+    I: Interner,
+    T: TypeFoldable<I>,
+{
+    let mut orig_values = Default::default();
+    let canonical =
+        Canonicalizer::canonicalize_response(delegate, max_input_universe, &mut orig_values, value);
+    canonical
+}
+
+/// After calling a canonical query, we apply the constraints returned
+/// by the query using this function.
+///
+/// This happens in three steps:
+/// - we instantiate the bound variables of the query response
+/// - we unify the `var_values` of the response with the `original_values`
+/// - we apply the `external_constraints` returned by the query, returning
+///   the `normalization_nested_goals`
+pub(super) fn instantiate_and_apply_query_response<D, I>(
+    delegate: &D,
+    param_env: I::ParamEnv,
+    original_values: &[I::GenericArg],
+    response: CanonicalResponse<I>,
+    span: I::Span,
+) -> (NestedNormalizationGoals<I>, Certainty)
+where
+    D: SolverDelegate<Interner = I>,
+    I: Interner,
+{
+    let instantiation =
+        compute_query_response_instantiation_values(delegate, &original_values, &response, span);
+
+    let Response { var_values, external_constraints, certainty } =
+        delegate.instantiate_canonical(response, instantiation);
+
+    unify_query_var_values(delegate, param_env, &original_values, var_values, span);
+
+    let ExternalConstraintsData { region_constraints, opaque_types, normalization_nested_goals } =
+        &*external_constraints;
+
+    register_region_constraints(delegate, region_constraints, span);
+    register_new_opaque_types(delegate, opaque_types, span);
+
+    (normalization_nested_goals.clone(), certainty)
+}
+
+/// This returns the canonical variable values to instantiate the bound variables of
+/// the canonical response. This depends on the `original_values` for the
+/// bound variables.
+fn compute_query_response_instantiation_values<D, I, T>(
+    delegate: &D,
+    original_values: &[I::GenericArg],
+    response: &Canonical<I, T>,
+    span: I::Span,
+) -> CanonicalVarValues<I>
+where
+    D: SolverDelegate<Interner = I>,
+    I: Interner,
+    T: ResponseT<I>,
+{
+    // FIXME: Longterm canonical queries should deal with all placeholders
+    // created inside of the query directly instead of returning them to the
+    // caller.
+    let prev_universe = delegate.universe();
+    let universes_created_in_query = response.max_universe.index();
+    for _ in 0..universes_created_in_query {
+        delegate.create_next_universe();
+    }
+
+    let var_values = response.value.var_values();
+    assert_eq!(original_values.len(), var_values.len());
+
+    // If the query did not make progress with constraining inference variables,
+    // we would normally create a new inference variables for bound existential variables
+    // only then unify this new inference variable with the inference variable from
+    // the input.
+    //
+    // We therefore instantiate the existential variable in the canonical response with the
+    // inference variable of the input right away, which is more performant.
+    let mut opt_values = IndexVec::from_elem_n(None, response.variables.len());
+    for (original_value, result_value) in iter::zip(original_values, var_values.var_values.iter()) {
+        match result_value.kind() {
+            ty::GenericArgKind::Type(t) => {
+                // We disable the instantiation guess for inference variables
+                // and only use it for placeholders. We need to handle the
+                // `sub_root` of type inference variables which would make this
+                // more involved. They are also a lot rarer than region variables.
+                if let ty::Bound(debruijn, b) = t.kind()
+                    && !matches!(
+                        response.variables.get(b.var().as_usize()).unwrap(),
+                        CanonicalVarKind::Ty { .. }
+                    )
+                {
+                    assert_eq!(debruijn, ty::INNERMOST);
+                    opt_values[b.var()] = Some(*original_value);
+                }
+            }
+            ty::GenericArgKind::Lifetime(r) => {
+                if let ty::ReBound(debruijn, br) = r.kind() {
+                    assert_eq!(debruijn, ty::INNERMOST);
+                    opt_values[br.var()] = Some(*original_value);
+                }
+            }
+            ty::GenericArgKind::Const(c) => {
+                if let ty::ConstKind::Bound(debruijn, bv) = c.kind() {
+                    assert_eq!(debruijn, ty::INNERMOST);
+                    opt_values[bv.var()] = Some(*original_value);
+                }
+            }
+        }
+    }
+    CanonicalVarValues::instantiate(delegate.cx(), response.variables, |var_values, kind| {
+        if kind.universe() != ty::UniverseIndex::ROOT {
+            // A variable from inside a binder of the query. While ideally these shouldn't
+            // exist at all (see the FIXME at the start of this method), we have to deal with
+            // them for now.
+            delegate.instantiate_canonical_var(kind, span, &var_values, |idx| {
+                prev_universe + idx.index()
+            })
+        } else if kind.is_existential() {
+            // As an optimization we sometimes avoid creating a new inference variable here.
+            //
+            // All new inference variables we create start out in the current universe of the caller.
+            // This is conceptually wrong as these inference variables would be able to name
+            // more placeholders then they should be able to. However the inference variables have
+            // to "come from somewhere", so by equating them with the original values of the caller
+            // later on, we pull them down into their correct universe again.
+            if let Some(v) = opt_values[ty::BoundVar::from_usize(var_values.len())] {
+                v
+            } else {
+                delegate.instantiate_canonical_var(kind, span, &var_values, |_| prev_universe)
+            }
+        } else {
+            // For placeholders which were already part of the input, we simply map this
+            // universal bound variable back the placeholder of the input.
+            original_values[kind.expect_placeholder_index()]
+        }
+    })
+}
+
+/// Unify the `original_values` with the `var_values` returned by the canonical query..
+///
+/// This assumes that this unification will always succeed. This is the case when
+/// applying a query response right away. However, calling a canonical query, doing any
+/// other kind of trait solving, and only then instantiating the result of the query
+/// can cause the instantiation to fail. This is not supported and we ICE in this case.
+///
+/// We always structurally instantiate aliases. Relating aliases needs to be different
+/// depending on whether the alias is *rigid* or not. We're only really able to tell
+/// whether an alias is rigid by using the trait solver. When instantiating a response
+/// from the solver we assume that the solver correctly handled aliases and therefore
+/// always relate them structurally here.
+#[instrument(level = "trace", skip(delegate))]
+fn unify_query_var_values<D, I>(
+    delegate: &D,
+    param_env: I::ParamEnv,
+    original_values: &[I::GenericArg],
+    var_values: CanonicalVarValues<I>,
+    span: I::Span,
+) where
+    D: SolverDelegate<Interner = I>,
+    I: Interner,
+{
+    assert_eq!(original_values.len(), var_values.len());
+
+    for (&orig, response) in iter::zip(original_values, var_values.var_values.iter()) {
+        let goals =
+            delegate.eq_structurally_relating_aliases(param_env, orig, response, span).unwrap();
+        assert!(goals.is_empty());
+    }
+}
+
+fn register_region_constraints<D, I>(
+    delegate: &D,
+    outlives: &[ty::OutlivesPredicate<I, I::GenericArg>],
+    span: I::Span,
+) where
+    D: SolverDelegate<Interner = I>,
+    I: Interner,
+{
+    for &ty::OutlivesPredicate(lhs, rhs) in outlives {
+        match lhs.kind() {
+            ty::GenericArgKind::Lifetime(lhs) => delegate.sub_regions(rhs, lhs, span),
+            ty::GenericArgKind::Type(lhs) => delegate.register_ty_outlives(lhs, rhs, span),
+            ty::GenericArgKind::Const(_) => panic!("const outlives: {lhs:?}: {rhs:?}"),
+        }
+    }
+}
+
+fn register_new_opaque_types<D, I>(
+    delegate: &D,
+    opaque_types: &[(ty::OpaqueTypeKey<I>, I::Ty)],
+    span: I::Span,
+) where
+    D: SolverDelegate<Interner = I>,
+    I: Interner,
+{
+    for &(key, ty) in opaque_types {
+        let prev = delegate.register_hidden_type_in_storage(key, ty, span);
+        // We eagerly resolve inference variables when computing the query response.
+        // This can cause previously distinct opaque type keys to now be structurally equal.
+        //
+        // To handle this, we store any duplicate entries in a separate list to check them
+        // at the end of typeck/borrowck. We could alternatively eagerly equate the hidden
+        // types here. However, doing so is difficult as it may result in nested goals and
+        // any errors may make it harder to track the control flow for diagnostics.
+        if let Some(prev) = prev {
+            delegate.add_duplicate_opaque_type(key, prev, span);
+        }
+    }
+}
+
+/// Used by proof trees to be able to recompute intermediate actions while
+/// evaluating a goal. The `var_values` not only include the bound variables
+/// of the query input, but also contain all unconstrained inference vars
+/// created while evaluating this goal.
+pub fn make_canonical_state<D, I, T>(
+    delegate: &D,
+    var_values: &[I::GenericArg],
+    max_input_universe: ty::UniverseIndex,
+    data: T,
+) -> inspect::CanonicalState<I, T>
+where
+    D: SolverDelegate<Interner = I>,
+    I: Interner,
+    T: TypeFoldable<I>,
+{
+    let var_values = CanonicalVarValues { var_values: delegate.cx().mk_args(var_values) };
+    let state = inspect::State { var_values, data };
+    let state = eager_resolve_vars(delegate, state);
+    Canonicalizer::canonicalize_response(delegate, max_input_universe, &mut vec![], state)
+}
+
+// FIXME: needs to be pub to be accessed by downstream
+// `rustc_trait_selection::solve::inspect::analyse`.
+pub fn instantiate_canonical_state<D, I, T>(
+    delegate: &D,
+    span: I::Span,
+    param_env: I::ParamEnv,
+    orig_values: &mut Vec<I::GenericArg>,
+    state: inspect::CanonicalState<I, T>,
+) -> T
+where
+    D: SolverDelegate<Interner = I>,
+    I: Interner,
+    T: TypeFoldable<I>,
+{
+    // In case any fresh inference variables have been created between `state`
+    // and the previous instantiation, extend `orig_values` for it.
+    orig_values.extend(
+        state.value.var_values.var_values.as_slice()[orig_values.len()..]
+            .iter()
+            .map(|&arg| delegate.fresh_var_for_kind_with_span(arg, span)),
+    );
+
+    let instantiation =
+        compute_query_response_instantiation_values(delegate, orig_values, &state, span);
+
+    let inspect::State { var_values, data } = delegate.instantiate_canonical(state, instantiation);
+
+    unify_query_var_values(delegate, param_env, orig_values, var_values, span);
+    data
+}
+
+pub fn response_no_constraints_raw<I: Interner>(
+    cx: I,
+    max_universe: ty::UniverseIndex,
+    variables: I::CanonicalVarKinds,
+    certainty: Certainty,
+) -> CanonicalResponse<I> {
+    ty::Canonical {
+        max_universe,
+        variables,
+        value: Response {
+            var_values: ty::CanonicalVarValues::make_identity(cx, variables),
+            // FIXME: maybe we should store the "no response" version in cx, like
+            // we do for cx.types and stuff.
+            external_constraints: cx.mk_external_constraints(ExternalConstraintsData::default()),
+            certainty,
+        },
+    }
+}
diff --git a/compiler/rustc_next_trait_solver/src/lib.rs b/compiler/rustc_next_trait_solver/src/lib.rs
index d3965e14c68..5fa29b7d9f8 100644
--- a/compiler/rustc_next_trait_solver/src/lib.rs
+++ b/compiler/rustc_next_trait_solver/src/lib.rs
@@ -10,7 +10,7 @@
 #![allow(rustc::usage_of_type_ir_traits)]
 // tidy-alphabetical-end
 
-pub mod canonicalizer;
+pub mod canonical;
 pub mod coherence;
 pub mod delegate;
 pub mod placeholder;
diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/canonical.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/canonical.rs
deleted file mode 100644
index 889588afe61..00000000000
--- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/canonical.rs
+++ /dev/null
@@ -1,517 +0,0 @@
-//! Canonicalization is used to separate some goal from its context,
-//! throwing away unnecessary information in the process.
-//!
-//! This is necessary to cache goals containing inference variables
-//! and placeholders without restricting them to the current `InferCtxt`.
-//!
-//! Canonicalization is fairly involved, for more details see the relevant
-//! section of the [rustc-dev-guide][c].
-//!
-//! [c]: https://rustc-dev-guide.rust-lang.org/solve/canonicalization.html
-
-use std::iter;
-
-use rustc_index::IndexVec;
-use rustc_type_ir::data_structures::HashSet;
-use rustc_type_ir::inherent::*;
-use rustc_type_ir::relate::solver_relating::RelateExt;
-use rustc_type_ir::solve::OpaqueTypesJank;
-use rustc_type_ir::{
-    self as ty, Canonical, CanonicalVarKind, CanonicalVarValues, InferCtxtLike, Interner,
-    TypeFoldable,
-};
-use tracing::{debug, instrument, trace};
-
-use crate::canonicalizer::Canonicalizer;
-use crate::delegate::SolverDelegate;
-use crate::resolve::eager_resolve_vars;
-use crate::solve::eval_ctxt::CurrentGoalKind;
-use crate::solve::{
-    CanonicalInput, CanonicalResponse, Certainty, EvalCtxt, ExternalConstraintsData, Goal,
-    MaybeCause, NestedNormalizationGoals, NoSolution, PredefinedOpaquesData, QueryInput,
-    QueryResult, Response, inspect, response_no_constraints_raw,
-};
-
-trait ResponseT<I: Interner> {
-    fn var_values(&self) -> CanonicalVarValues<I>;
-}
-
-impl<I: Interner> ResponseT<I> for Response<I> {
-    fn var_values(&self) -> CanonicalVarValues<I> {
-        self.var_values
-    }
-}
-
-impl<I: Interner, T> ResponseT<I> for inspect::State<I, T> {
-    fn var_values(&self) -> CanonicalVarValues<I> {
-        self.var_values
-    }
-}
-
-impl<D, I> EvalCtxt<'_, D>
-where
-    D: SolverDelegate<Interner = I>,
-    I: Interner,
-{
-    /// Canonicalizes the goal remembering the original values
-    /// for each bound variable.
-    ///
-    /// This expects `goal` and `opaque_types` to be eager resolved.
-    pub(super) fn canonicalize_goal(
-        delegate: &D,
-        goal: Goal<I, I::Predicate>,
-        opaque_types: Vec<(ty::OpaqueTypeKey<I>, I::Ty)>,
-    ) -> (Vec<I::GenericArg>, CanonicalInput<I, I::Predicate>) {
-        let mut orig_values = Default::default();
-        let canonical = Canonicalizer::canonicalize_input(
-            delegate,
-            &mut orig_values,
-            QueryInput {
-                goal,
-                predefined_opaques_in_body: delegate
-                    .cx()
-                    .mk_predefined_opaques_in_body(PredefinedOpaquesData { opaque_types }),
-            },
-        );
-        let query_input =
-            ty::CanonicalQueryInput { canonical, typing_mode: delegate.typing_mode() };
-        (orig_values, query_input)
-    }
-
-    /// To return the constraints of a canonical query to the caller, we canonicalize:
-    ///
-    /// - `var_values`: a map from bound variables in the canonical goal to
-    ///   the values inferred while solving the instantiated goal.
-    /// - `external_constraints`: additional constraints which aren't expressible
-    ///   using simple unification of inference variables.
-    ///
-    /// This takes the `shallow_certainty` which represents whether we're confident
-    /// that the final result of the current goal only depends on the nested goals.
-    ///
-    /// In case this is `Certainty::Maybe`, there may still be additional nested goals
-    /// or inference constraints required for this candidate to be hold. The candidate
-    /// always requires all already added constraints and nested goals.
-    #[instrument(level = "trace", skip(self), ret)]
-    pub(in crate::solve) fn evaluate_added_goals_and_make_canonical_response(
-        &mut self,
-        shallow_certainty: Certainty,
-    ) -> QueryResult<I> {
-        self.inspect.make_canonical_response(shallow_certainty);
-
-        let goals_certainty = self.try_evaluate_added_goals()?;
-        assert_eq!(
-            self.tainted,
-            Ok(()),
-            "EvalCtxt is tainted -- nested goals may have been dropped in a \
-            previous call to `try_evaluate_added_goals!`"
-        );
-
-        // We only check for leaks from universes which were entered inside
-        // of the query.
-        self.delegate.leak_check(self.max_input_universe).map_err(|NoSolution| {
-            trace!("failed the leak check");
-            NoSolution
-        })?;
-
-        let (certainty, normalization_nested_goals) =
-            match (self.current_goal_kind, shallow_certainty) {
-                // When normalizing, we've replaced the expected term with an unconstrained
-                // inference variable. This means that we dropped information which could
-                // have been important. We handle this by instead returning the nested goals
-                // to the caller, where they are then handled. We only do so if we do not
-                // need to recompute the `NormalizesTo` goal afterwards to avoid repeatedly
-                // uplifting its nested goals. This is the case if the `shallow_certainty` is
-                // `Certainty::Yes`.
-                (CurrentGoalKind::NormalizesTo, Certainty::Yes) => {
-                    let goals = std::mem::take(&mut self.nested_goals);
-                    // As we return all ambiguous nested goals, we can ignore the certainty
-                    // returned by `self.try_evaluate_added_goals()`.
-                    if goals.is_empty() {
-                        assert!(matches!(goals_certainty, Certainty::Yes));
-                    }
-                    (
-                        Certainty::Yes,
-                        NestedNormalizationGoals(
-                            goals.into_iter().map(|(s, g, _)| (s, g)).collect(),
-                        ),
-                    )
-                }
-                _ => {
-                    let certainty = shallow_certainty.and(goals_certainty);
-                    (certainty, NestedNormalizationGoals::empty())
-                }
-            };
-
-        if let Certainty::Maybe {
-            cause: cause @ MaybeCause::Overflow { keep_constraints: false, .. },
-            opaque_types_jank,
-        } = certainty
-        {
-            // If we have overflow, it's probable that we're substituting a type
-            // into itself infinitely and any partial substitutions in the query
-            // response are probably not useful anyways, so just return an empty
-            // query response.
-            //
-            // This may prevent us from potentially useful inference, e.g.
-            // 2 candidates, one ambiguous and one overflow, which both
-            // have the same inference constraints.
-            //
-            // Changing this to retain some constraints in the future
-            // won't be a breaking change, so this is good enough for now.
-            return Ok(self.make_ambiguous_response_no_constraints(cause, opaque_types_jank));
-        }
-
-        let external_constraints =
-            self.compute_external_query_constraints(certainty, normalization_nested_goals);
-        let (var_values, mut external_constraints) =
-            eager_resolve_vars(self.delegate, (self.var_values, external_constraints));
-
-        // Remove any trivial or duplicated region constraints once we've resolved regions
-        let mut unique = HashSet::default();
-        external_constraints.region_constraints.retain(|outlives| {
-            outlives.0.as_region().is_none_or(|re| re != outlives.1) && unique.insert(*outlives)
-        });
-
-        let canonical = Canonicalizer::canonicalize_response(
-            self.delegate,
-            self.max_input_universe,
-            &mut Default::default(),
-            Response {
-                var_values,
-                certainty,
-                external_constraints: self.cx().mk_external_constraints(external_constraints),
-            },
-        );
-
-        // HACK: We bail with overflow if the response would have too many non-region
-        // inference variables. This tends to only happen if we encounter a lot of
-        // ambiguous alias types which get replaced with fresh inference variables
-        // during generalization. This prevents hangs caused by an exponential blowup,
-        // see tests/ui/traits/next-solver/coherence-alias-hang.rs.
-        match self.current_goal_kind {
-            // We don't do so for `NormalizesTo` goals as we erased the expected term and
-            // bailing with overflow here would prevent us from detecting a type-mismatch,
-            // causing a coherence error in diesel, see #131969. We still bail with overflow
-            // when later returning from the parent AliasRelate goal.
-            CurrentGoalKind::NormalizesTo => {}
-            CurrentGoalKind::Misc | CurrentGoalKind::CoinductiveTrait => {
-                let num_non_region_vars = canonical
-                    .variables
-                    .iter()
-                    .filter(|c| !c.is_region() && c.is_existential())
-                    .count();
-                if num_non_region_vars > self.cx().recursion_limit() {
-                    debug!(?num_non_region_vars, "too many inference variables -> overflow");
-                    return Ok(self.make_ambiguous_response_no_constraints(
-                        MaybeCause::Overflow {
-                            suggest_increasing_limit: true,
-                            keep_constraints: false,
-                        },
-                        OpaqueTypesJank::AllGood,
-                    ));
-                }
-            }
-        }
-
-        Ok(canonical)
-    }
-
-    /// Constructs a totally unconstrained, ambiguous response to a goal.
-    ///
-    /// Take care when using this, since often it's useful to respond with
-    /// ambiguity but return constrained variables to guide inference.
-    pub(in crate::solve) fn make_ambiguous_response_no_constraints(
-        &self,
-        cause: MaybeCause,
-        opaque_types_jank: OpaqueTypesJank,
-    ) -> CanonicalResponse<I> {
-        response_no_constraints_raw(
-            self.cx(),
-            self.max_input_universe,
-            self.variables,
-            Certainty::Maybe { cause, opaque_types_jank },
-        )
-    }
-
-    /// Computes the region constraints and *new* opaque types registered when
-    /// proving a goal.
-    ///
-    /// If an opaque was already constrained before proving this goal, then the
-    /// external constraints do not need to record that opaque, since if it is
-    /// further constrained by inference, that will be passed back in the var
-    /// values.
-    #[instrument(level = "trace", skip(self), ret)]
-    fn compute_external_query_constraints(
-        &self,
-        certainty: Certainty,
-        normalization_nested_goals: NestedNormalizationGoals<I>,
-    ) -> ExternalConstraintsData<I> {
-        // We only return region constraints once the certainty is `Yes`. This
-        // is necessary as we may drop nested goals on ambiguity, which may result
-        // in unconstrained inference variables in the region constraints. It also
-        // prevents us from emitting duplicate region constraints, avoiding some
-        // unnecessary work. This slightly weakens the leak check in case it uses
-        // region constraints from an ambiguous nested goal. This is tested in both
-        // `tests/ui/higher-ranked/leak-check/leak-check-in-selection-5-ambig.rs` and
-        // `tests/ui/higher-ranked/leak-check/leak-check-in-selection-6-ambig-unify.rs`.
-        let region_constraints = if certainty == Certainty::Yes {
-            self.delegate.make_deduplicated_outlives_constraints()
-        } else {
-            Default::default()
-        };
-
-        // We only return *newly defined* opaque types from canonical queries.
-        //
-        // Constraints for any existing opaque types are already tracked by changes
-        // to the `var_values`.
-        let opaque_types = self
-            .delegate
-            .clone_opaque_types_added_since(self.initial_opaque_types_storage_num_entries);
-
-        ExternalConstraintsData { region_constraints, opaque_types, normalization_nested_goals }
-    }
-
-    /// After calling a canonical query, we apply the constraints returned
-    /// by the query using this function.
-    ///
-    /// This happens in three steps:
-    /// - we instantiate the bound variables of the query response
-    /// - we unify the `var_values` of the response with the `original_values`
-    /// - we apply the `external_constraints` returned by the query, returning
-    ///   the `normalization_nested_goals`
-    pub(super) fn instantiate_and_apply_query_response(
-        delegate: &D,
-        param_env: I::ParamEnv,
-        original_values: &[I::GenericArg],
-        response: CanonicalResponse<I>,
-        span: I::Span,
-    ) -> (NestedNormalizationGoals<I>, Certainty) {
-        let instantiation = Self::compute_query_response_instantiation_values(
-            delegate,
-            &original_values,
-            &response,
-            span,
-        );
-
-        let Response { var_values, external_constraints, certainty } =
-            delegate.instantiate_canonical(response, instantiation);
-
-        Self::unify_query_var_values(delegate, param_env, &original_values, var_values, span);
-
-        let ExternalConstraintsData {
-            region_constraints,
-            opaque_types,
-            normalization_nested_goals,
-        } = &*external_constraints;
-
-        Self::register_region_constraints(delegate, region_constraints, span);
-        Self::register_new_opaque_types(delegate, opaque_types, span);
-
-        (normalization_nested_goals.clone(), certainty)
-    }
-
-    /// This returns the canonical variable values to instantiate the bound variables of
-    /// the canonical response. This depends on the `original_values` for the
-    /// bound variables.
-    fn compute_query_response_instantiation_values<T: ResponseT<I>>(
-        delegate: &D,
-        original_values: &[I::GenericArg],
-        response: &Canonical<I, T>,
-        span: I::Span,
-    ) -> CanonicalVarValues<I> {
-        // FIXME: Longterm canonical queries should deal with all placeholders
-        // created inside of the query directly instead of returning them to the
-        // caller.
-        let prev_universe = delegate.universe();
-        let universes_created_in_query = response.max_universe.index();
-        for _ in 0..universes_created_in_query {
-            delegate.create_next_universe();
-        }
-
-        let var_values = response.value.var_values();
-        assert_eq!(original_values.len(), var_values.len());
-
-        // If the query did not make progress with constraining inference variables,
-        // we would normally create a new inference variables for bound existential variables
-        // only then unify this new inference variable with the inference variable from
-        // the input.
-        //
-        // We therefore instantiate the existential variable in the canonical response with the
-        // inference variable of the input right away, which is more performant.
-        let mut opt_values = IndexVec::from_elem_n(None, response.variables.len());
-        for (original_value, result_value) in
-            iter::zip(original_values, var_values.var_values.iter())
-        {
-            match result_value.kind() {
-                ty::GenericArgKind::Type(t) => {
-                    // We disable the instantiation guess for inference variables
-                    // and only use it for placeholders. We need to handle the
-                    // `sub_root` of type inference variables which would make this
-                    // more involved. They are also a lot rarer than region variables.
-                    if let ty::Bound(debruijn, b) = t.kind()
-                        && !matches!(
-                            response.variables.get(b.var().as_usize()).unwrap(),
-                            CanonicalVarKind::Ty { .. }
-                        )
-                    {
-                        assert_eq!(debruijn, ty::INNERMOST);
-                        opt_values[b.var()] = Some(*original_value);
-                    }
-                }
-                ty::GenericArgKind::Lifetime(r) => {
-                    if let ty::ReBound(debruijn, br) = r.kind() {
-                        assert_eq!(debruijn, ty::INNERMOST);
-                        opt_values[br.var()] = Some(*original_value);
-                    }
-                }
-                ty::GenericArgKind::Const(c) => {
-                    if let ty::ConstKind::Bound(debruijn, bv) = c.kind() {
-                        assert_eq!(debruijn, ty::INNERMOST);
-                        opt_values[bv.var()] = Some(*original_value);
-                    }
-                }
-            }
-        }
-        CanonicalVarValues::instantiate(delegate.cx(), response.variables, |var_values, kind| {
-            if kind.universe() != ty::UniverseIndex::ROOT {
-                // A variable from inside a binder of the query. While ideally these shouldn't
-                // exist at all (see the FIXME at the start of this method), we have to deal with
-                // them for now.
-                delegate.instantiate_canonical_var(kind, span, &var_values, |idx| {
-                    prev_universe + idx.index()
-                })
-            } else if kind.is_existential() {
-                // As an optimization we sometimes avoid creating a new inference variable here.
-                //
-                // All new inference variables we create start out in the current universe of the caller.
-                // This is conceptually wrong as these inference variables would be able to name
-                // more placeholders then they should be able to. However the inference variables have
-                // to "come from somewhere", so by equating them with the original values of the caller
-                // later on, we pull them down into their correct universe again.
-                if let Some(v) = opt_values[ty::BoundVar::from_usize(var_values.len())] {
-                    v
-                } else {
-                    delegate.instantiate_canonical_var(kind, span, &var_values, |_| prev_universe)
-                }
-            } else {
-                // For placeholders which were already part of the input, we simply map this
-                // universal bound variable back the placeholder of the input.
-                original_values[kind.expect_placeholder_index()]
-            }
-        })
-    }
-
-    /// Unify the `original_values` with the `var_values` returned by the canonical query..
-    ///
-    /// This assumes that this unification will always succeed. This is the case when
-    /// applying a query response right away. However, calling a canonical query, doing any
-    /// other kind of trait solving, and only then instantiating the result of the query
-    /// can cause the instantiation to fail. This is not supported and we ICE in this case.
-    ///
-    /// We always structurally instantiate aliases. Relating aliases needs to be different
-    /// depending on whether the alias is *rigid* or not. We're only really able to tell
-    /// whether an alias is rigid by using the trait solver. When instantiating a response
-    /// from the solver we assume that the solver correctly handled aliases and therefore
-    /// always relate them structurally here.
-    #[instrument(level = "trace", skip(delegate))]
-    fn unify_query_var_values(
-        delegate: &D,
-        param_env: I::ParamEnv,
-        original_values: &[I::GenericArg],
-        var_values: CanonicalVarValues<I>,
-        span: I::Span,
-    ) {
-        assert_eq!(original_values.len(), var_values.len());
-
-        for (&orig, response) in iter::zip(original_values, var_values.var_values.iter()) {
-            let goals =
-                delegate.eq_structurally_relating_aliases(param_env, orig, response, span).unwrap();
-            assert!(goals.is_empty());
-        }
-    }
-
-    fn register_region_constraints(
-        delegate: &D,
-        outlives: &[ty::OutlivesPredicate<I, I::GenericArg>],
-        span: I::Span,
-    ) {
-        for &ty::OutlivesPredicate(lhs, rhs) in outlives {
-            match lhs.kind() {
-                ty::GenericArgKind::Lifetime(lhs) => delegate.sub_regions(rhs, lhs, span),
-                ty::GenericArgKind::Type(lhs) => delegate.register_ty_outlives(lhs, rhs, span),
-                ty::GenericArgKind::Const(_) => panic!("const outlives: {lhs:?}: {rhs:?}"),
-            }
-        }
-    }
-
-    fn register_new_opaque_types(
-        delegate: &D,
-        opaque_types: &[(ty::OpaqueTypeKey<I>, I::Ty)],
-        span: I::Span,
-    ) {
-        for &(key, ty) in opaque_types {
-            let prev = delegate.register_hidden_type_in_storage(key, ty, span);
-            // We eagerly resolve inference variables when computing the query response.
-            // This can cause previously distinct opaque type keys to now be structurally equal.
-            //
-            // To handle this, we store any duplicate entries in a separate list to check them
-            // at the end of typeck/borrowck. We could alternatively eagerly equate the hidden
-            // types here. However, doing so is difficult as it may result in nested goals and
-            // any errors may make it harder to track the control flow for diagnostics.
-            if let Some(prev) = prev {
-                delegate.add_duplicate_opaque_type(key, prev, span);
-            }
-        }
-    }
-}
-
-/// Used by proof trees to be able to recompute intermediate actions while
-/// evaluating a goal. The `var_values` not only include the bound variables
-/// of the query input, but also contain all unconstrained inference vars
-/// created while evaluating this goal.
-pub(in crate::solve) fn make_canonical_state<D, T, I>(
-    delegate: &D,
-    var_values: &[I::GenericArg],
-    max_input_universe: ty::UniverseIndex,
-    data: T,
-) -> inspect::CanonicalState<I, T>
-where
-    D: SolverDelegate<Interner = I>,
-    I: Interner,
-    T: TypeFoldable<I>,
-{
-    let var_values = CanonicalVarValues { var_values: delegate.cx().mk_args(var_values) };
-    let state = inspect::State { var_values, data };
-    let state = eager_resolve_vars(delegate, state);
-    Canonicalizer::canonicalize_response(delegate, max_input_universe, &mut vec![], state)
-}
-
-// FIXME: needs to be pub to be accessed by downstream
-// `rustc_trait_selection::solve::inspect::analyse`.
-pub fn instantiate_canonical_state<D, I, T: TypeFoldable<I>>(
-    delegate: &D,
-    span: I::Span,
-    param_env: I::ParamEnv,
-    orig_values: &mut Vec<I::GenericArg>,
-    state: inspect::CanonicalState<I, T>,
-) -> T
-where
-    D: SolverDelegate<Interner = I>,
-    I: Interner,
-{
-    // In case any fresh inference variables have been created between `state`
-    // and the previous instantiation, extend `orig_values` for it.
-    orig_values.extend(
-        state.value.var_values.var_values.as_slice()[orig_values.len()..]
-            .iter()
-            .map(|&arg| delegate.fresh_var_for_kind_with_span(arg, span)),
-    );
-
-    let instantiation =
-        EvalCtxt::compute_query_response_instantiation_values(delegate, orig_values, &state, span);
-
-    let inspect::State { var_values, data } = delegate.instantiate_canonical(state, instantiation);
-
-    EvalCtxt::unify_query_var_values(delegate, param_env, orig_values, var_values, span);
-    data
-}
diff --git a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs
index 5df7c92d881..bb86357a85f 100644
--- a/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs
+++ b/compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs
@@ -17,6 +17,10 @@ use rustc_type_ir::{
 use tracing::{debug, instrument, trace};
 
 use super::has_only_region_constraints;
+use crate::canonical::{
+    canonicalize_goal, canonicalize_response, instantiate_and_apply_query_response,
+    response_no_constraints_raw,
+};
 use crate::coherence;
 use crate::delegate::SolverDelegate;
 use crate::placeholder::BoundVarReplacer;
@@ -24,12 +28,11 @@ use crate::resolve::eager_resolve_vars;
 use crate::solve::search_graph::SearchGraph;
 use crate::solve::ty::may_use_unstable_feature;
 use crate::solve::{
-    CanonicalInput, Certainty, FIXPOINT_STEP_LIMIT, Goal, GoalEvaluation, GoalSource,
-    GoalStalledOn, HasChanged, NestedNormalizationGoals, NoSolution, QueryInput, QueryResult,
-    inspect,
+    CanonicalInput, CanonicalResponse, Certainty, ExternalConstraintsData, FIXPOINT_STEP_LIMIT,
+    Goal, GoalEvaluation, GoalSource, GoalStalledOn, HasChanged, MaybeCause,
+    NestedNormalizationGoals, NoSolution, QueryInput, QueryResult, Response, inspect,
 };
 
-pub(super) mod canonical;
 mod probe;
 
 /// The kind of goal we're currently proving.
@@ -464,8 +467,7 @@ where
         let opaque_types = self.delegate.clone_opaque_types_lookup_table();
         let (goal, opaque_types) = eager_resolve_vars(self.delegate, (goal, opaque_types));
 
-        let (orig_values, canonical_goal) =
-            Self::canonicalize_goal(self.delegate, goal, opaque_types);
+        let (orig_values, canonical_goal) = canonicalize_goal(self.delegate, goal, opaque_types);
         let canonical_result = self.search_graph.evaluate_goal(
             self.cx(),
             canonical_goal,
@@ -480,7 +482,7 @@ where
         let has_changed =
             if !has_only_region_constraints(response) { HasChanged::Yes } else { HasChanged::No };
 
-        let (normalization_nested_goals, certainty) = Self::instantiate_and_apply_query_response(
+        let (normalization_nested_goals, certainty) = instantiate_and_apply_query_response(
             self.delegate,
             goal.param_env,
             &orig_values,
@@ -1223,6 +1225,198 @@ where
             vec![]
         }
     }
+
+    /// To return the constraints of a canonical query to the caller, we canonicalize:
+    ///
+    /// - `var_values`: a map from bound variables in the canonical goal to
+    ///   the values inferred while solving the instantiated goal.
+    /// - `external_constraints`: additional constraints which aren't expressible
+    ///   using simple unification of inference variables.
+    ///
+    /// This takes the `shallow_certainty` which represents whether we're confident
+    /// that the final result of the current goal only depends on the nested goals.
+    ///
+    /// In case this is `Certainty::Maybe`, there may still be additional nested goals
+    /// or inference constraints required for this candidate to be hold. The candidate
+    /// always requires all already added constraints and nested goals.
+    #[instrument(level = "trace", skip(self), ret)]
+    pub(in crate::solve) fn evaluate_added_goals_and_make_canonical_response(
+        &mut self,
+        shallow_certainty: Certainty,
+    ) -> QueryResult<I> {
+        self.inspect.make_canonical_response(shallow_certainty);
+
+        let goals_certainty = self.try_evaluate_added_goals()?;
+        assert_eq!(
+            self.tainted,
+            Ok(()),
+            "EvalCtxt is tainted -- nested goals may have been dropped in a \
+            previous call to `try_evaluate_added_goals!`"
+        );
+
+        // We only check for leaks from universes which were entered inside
+        // of the query.
+        self.delegate.leak_check(self.max_input_universe).map_err(|NoSolution| {
+            trace!("failed the leak check");
+            NoSolution
+        })?;
+
+        let (certainty, normalization_nested_goals) =
+            match (self.current_goal_kind, shallow_certainty) {
+                // When normalizing, we've replaced the expected term with an unconstrained
+                // inference variable. This means that we dropped information which could
+                // have been important. We handle this by instead returning the nested goals
+                // to the caller, where they are then handled. We only do so if we do not
+                // need to recompute the `NormalizesTo` goal afterwards to avoid repeatedly
+                // uplifting its nested goals. This is the case if the `shallow_certainty` is
+                // `Certainty::Yes`.
+                (CurrentGoalKind::NormalizesTo, Certainty::Yes) => {
+                    let goals = std::mem::take(&mut self.nested_goals);
+                    // As we return all ambiguous nested goals, we can ignore the certainty
+                    // returned by `self.try_evaluate_added_goals()`.
+                    if goals.is_empty() {
+                        assert!(matches!(goals_certainty, Certainty::Yes));
+                    }
+                    (
+                        Certainty::Yes,
+                        NestedNormalizationGoals(
+                            goals.into_iter().map(|(s, g, _)| (s, g)).collect(),
+                        ),
+                    )
+                }
+                _ => {
+                    let certainty = shallow_certainty.and(goals_certainty);
+                    (certainty, NestedNormalizationGoals::empty())
+                }
+            };
+
+        if let Certainty::Maybe {
+            cause: cause @ MaybeCause::Overflow { keep_constraints: false, .. },
+            opaque_types_jank,
+        } = certainty
+        {
+            // If we have overflow, it's probable that we're substituting a type
+            // into itself infinitely and any partial substitutions in the query
+            // response are probably not useful anyways, so just return an empty
+            // query response.
+            //
+            // This may prevent us from potentially useful inference, e.g.
+            // 2 candidates, one ambiguous and one overflow, which both
+            // have the same inference constraints.
+            //
+            // Changing this to retain some constraints in the future
+            // won't be a breaking change, so this is good enough for now.
+            return Ok(self.make_ambiguous_response_no_constraints(cause, opaque_types_jank));
+        }
+
+        let external_constraints =
+            self.compute_external_query_constraints(certainty, normalization_nested_goals);
+        let (var_values, mut external_constraints) =
+            eager_resolve_vars(self.delegate, (self.var_values, external_constraints));
+
+        // Remove any trivial or duplicated region constraints once we've resolved regions
+        let mut unique = HashSet::default();
+        external_constraints.region_constraints.retain(|outlives| {
+            outlives.0.as_region().is_none_or(|re| re != outlives.1) && unique.insert(*outlives)
+        });
+
+        let canonical = canonicalize_response(
+            self.delegate,
+            self.max_input_universe,
+            Response {
+                var_values,
+                certainty,
+                external_constraints: self.cx().mk_external_constraints(external_constraints),
+            },
+        );
+
+        // HACK: We bail with overflow if the response would have too many non-region
+        // inference variables. This tends to only happen if we encounter a lot of
+        // ambiguous alias types which get replaced with fresh inference variables
+        // during generalization. This prevents hangs caused by an exponential blowup,
+        // see tests/ui/traits/next-solver/coherence-alias-hang.rs.
+        match self.current_goal_kind {
+            // We don't do so for `NormalizesTo` goals as we erased the expected term and
+            // bailing with overflow here would prevent us from detecting a type-mismatch,
+            // causing a coherence error in diesel, see #131969. We still bail with overflow
+            // when later returning from the parent AliasRelate goal.
+            CurrentGoalKind::NormalizesTo => {}
+            CurrentGoalKind::Misc | CurrentGoalKind::CoinductiveTrait => {
+                let num_non_region_vars = canonical
+                    .variables
+                    .iter()
+                    .filter(|c| !c.is_region() && c.is_existential())
+                    .count();
+                if num_non_region_vars > self.cx().recursion_limit() {
+                    debug!(?num_non_region_vars, "too many inference variables -> overflow");
+                    return Ok(self.make_ambiguous_response_no_constraints(
+                        MaybeCause::Overflow {
+                            suggest_increasing_limit: true,
+                            keep_constraints: false,
+                        },
+                        OpaqueTypesJank::AllGood,
+                    ));
+                }
+            }
+        }
+
+        Ok(canonical)
+    }
+
+    /// Constructs a totally unconstrained, ambiguous response to a goal.
+    ///
+    /// Take care when using this, since often it's useful to respond with
+    /// ambiguity but return constrained variables to guide inference.
+    pub(in crate::solve) fn make_ambiguous_response_no_constraints(
+        &self,
+        cause: MaybeCause,
+        opaque_types_jank: OpaqueTypesJank,
+    ) -> CanonicalResponse<I> {
+        response_no_constraints_raw(
+            self.cx(),
+            self.max_input_universe,
+            self.variables,
+            Certainty::Maybe { cause, opaque_types_jank },
+        )
+    }
+
+    /// Computes the region constraints and *new* opaque types registered when
+    /// proving a goal.
+    ///
+    /// If an opaque was already constrained before proving this goal, then the
+    /// external constraints do not need to record that opaque, since if it is
+    /// further constrained by inference, that will be passed back in the var
+    /// values.
+    #[instrument(level = "trace", skip(self), ret)]
+    fn compute_external_query_constraints(
+        &self,
+        certainty: Certainty,
+        normalization_nested_goals: NestedNormalizationGoals<I>,
+    ) -> ExternalConstraintsData<I> {
+        // We only return region constraints once the certainty is `Yes`. This
+        // is necessary as we may drop nested goals on ambiguity, which may result
+        // in unconstrained inference variables in the region constraints. It also
+        // prevents us from emitting duplicate region constraints, avoiding some
+        // unnecessary work. This slightly weakens the leak check in case it uses
+        // region constraints from an ambiguous nested goal. This is tested in both
+        // `tests/ui/higher-ranked/leak-check/leak-check-in-selection-5-ambig.rs` and
+        // `tests/ui/higher-ranked/leak-check/leak-check-in-selection-6-ambig-unify.rs`.
+        let region_constraints = if certainty == Certainty::Yes {
+            self.delegate.make_deduplicated_outlives_constraints()
+        } else {
+            Default::default()
+        };
+
+        // We only return *newly defined* opaque types from canonical queries.
+        //
+        // Constraints for any existing opaque types are already tracked by changes
+        // to the `var_values`.
+        let opaque_types = self
+            .delegate
+            .clone_opaque_types_added_since(self.initial_opaque_types_storage_num_entries);
+
+        ExternalConstraintsData { region_constraints, opaque_types, normalization_nested_goals }
+    }
 }
 
 /// Eagerly replace aliases with inference variables, emitting `AliasRelate`
@@ -1363,7 +1557,7 @@ pub(super) fn evaluate_root_goal_for_proof_tree<D: SolverDelegate<Interner = I>,
     let opaque_types = delegate.clone_opaque_types_lookup_table();
     let (goal, opaque_types) = eager_resolve_vars(delegate, (goal, opaque_types));
 
-    let (orig_values, canonical_goal) = EvalCtxt::canonicalize_goal(delegate, goal, opaque_types);
+    let (orig_values, canonical_goal) = canonicalize_goal(delegate, goal, opaque_types);
 
     let (canonical_result, final_revision) =
         delegate.cx().evaluate_root_goal_for_proof_tree_raw(canonical_goal);
@@ -1380,7 +1574,7 @@ pub(super) fn evaluate_root_goal_for_proof_tree<D: SolverDelegate<Interner = I>,
         Ok(response) => response,
     };
 
-    let (normalization_nested_goals, _certainty) = EvalCtxt::instantiate_and_apply_query_response(
+    let (normalization_nested_goals, _certainty) = instantiate_and_apply_query_response(
         delegate,
         goal.param_env,
         &proof_tree.orig_values,
diff --git a/compiler/rustc_next_trait_solver/src/solve/inspect/build.rs b/compiler/rustc_next_trait_solver/src/solve/inspect/build.rs
index 2675ed0d0da..4369148baf9 100644
--- a/compiler/rustc_next_trait_solver/src/solve/inspect/build.rs
+++ b/compiler/rustc_next_trait_solver/src/solve/inspect/build.rs
@@ -10,8 +10,8 @@ use derive_where::derive_where;
 use rustc_type_ir::inherent::*;
 use rustc_type_ir::{self as ty, Interner};
 
+use crate::canonical;
 use crate::delegate::SolverDelegate;
-use crate::solve::eval_ctxt::canonical;
 use crate::solve::{Certainty, Goal, GoalSource, QueryResult, inspect};
 
 /// We need to know whether to build a prove tree while evaluating. We
diff --git a/compiler/rustc_next_trait_solver/src/solve/inspect/mod.rs b/compiler/rustc_next_trait_solver/src/solve/inspect/mod.rs
index 0d8c0060126..65f32f1947f 100644
--- a/compiler/rustc_next_trait_solver/src/solve/inspect/mod.rs
+++ b/compiler/rustc_next_trait_solver/src/solve/inspect/mod.rs
@@ -2,5 +2,3 @@ pub use rustc_type_ir::solve::inspect::*;
 
 mod build;
 pub(in crate::solve) use build::*;
-
-pub use crate::solve::eval_ctxt::canonical::instantiate_canonical_state;
diff --git a/compiler/rustc_next_trait_solver/src/solve/mod.rs b/compiler/rustc_next_trait_solver/src/solve/mod.rs
index fb900b592d1..afb86aaf8ab 100644
--- a/compiler/rustc_next_trait_solver/src/solve/mod.rs
+++ b/compiler/rustc_next_trait_solver/src/solve/mod.rs
@@ -380,25 +380,6 @@ where
     }
 }
 
-fn response_no_constraints_raw<I: Interner>(
-    cx: I,
-    max_universe: ty::UniverseIndex,
-    variables: I::CanonicalVarKinds,
-    certainty: Certainty,
-) -> CanonicalResponse<I> {
-    ty::Canonical {
-        max_universe,
-        variables,
-        value: Response {
-            var_values: ty::CanonicalVarValues::make_identity(cx, variables),
-            // FIXME: maybe we should store the "no response" version in cx, like
-            // we do for cx.types and stuff.
-            external_constraints: cx.mk_external_constraints(ExternalConstraintsData::default()),
-            certainty,
-        },
-    }
-}
-
 /// The result of evaluating a goal.
 pub struct GoalEvaluation<I: Interner> {
     /// The goal we've evaluated. This is the input goal, but potentially with its
diff --git a/compiler/rustc_next_trait_solver/src/solve/search_graph.rs b/compiler/rustc_next_trait_solver/src/solve/search_graph.rs
index 289325d7055..aa9dfc9a9a2 100644
--- a/compiler/rustc_next_trait_solver/src/solve/search_graph.rs
+++ b/compiler/rustc_next_trait_solver/src/solve/search_graph.rs
@@ -6,6 +6,7 @@ use rustc_type_ir::search_graph::{self, PathKind};
 use rustc_type_ir::solve::{CanonicalInput, Certainty, NoSolution, QueryResult};
 use rustc_type_ir::{Interner, TypingMode};
 
+use crate::canonical::response_no_constraints_raw;
 use crate::delegate::SolverDelegate;
 use crate::solve::{
     EvalCtxt, FIXPOINT_STEP_LIMIT, has_no_inference_or_external_constraints, inspect,
@@ -127,7 +128,7 @@ fn response_no_constraints<I: Interner>(
     input: CanonicalInput<I>,
     certainty: Certainty,
 ) -> QueryResult<I> {
-    Ok(super::response_no_constraints_raw(
+    Ok(response_no_constraints_raw(
         cx,
         input.canonical.max_universe,
         input.canonical.variables,
diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs
index 297df7c2c97..795cb2b2cfe 100644
--- a/compiler/rustc_session/src/config.rs
+++ b/compiler/rustc_session/src/config.rs
@@ -1509,6 +1509,11 @@ impl Options {
     pub fn get_symbol_mangling_version(&self) -> SymbolManglingVersion {
         self.cg.symbol_mangling_version.unwrap_or(SymbolManglingVersion::Legacy)
     }
+
+    #[inline]
+    pub fn autodiff_enabled(&self) -> bool {
+        self.unstable_opts.autodiff.contains(&AutoDiff::Enable)
+    }
 }
 
 impl UnstableOptions {
diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs
index 3525c7c1d1a..d0dd2cdac0c 100644
--- a/compiler/rustc_session/src/session.rs
+++ b/compiler/rustc_session/src/session.rs
@@ -600,6 +600,13 @@ impl Session {
 
     /// Calculates the flavor of LTO to use for this compilation.
     pub fn lto(&self) -> config::Lto {
+        // Autodiff currently requires fat-lto to have access to the llvm-ir of all (indirectly) used functions and types.
+        // fat-lto is the easiest solution to this requirement, but quite expensive.
+        // FIXME(autodiff): Make autodiff also work with embed-bc instead of fat-lto.
+        if self.opts.autodiff_enabled() {
+            return config::Lto::Fat;
+        }
+
         // If our target has codegen requirements ignore the command line
         if self.target.requires_lto {
             return config::Lto::Fat;
diff --git a/compiler/rustc_trait_selection/src/solve/inspect/analyse.rs b/compiler/rustc_trait_selection/src/solve/inspect/analyse.rs
index 086a7a44786..c010add0fc5 100644
--- a/compiler/rustc_trait_selection/src/solve/inspect/analyse.rs
+++ b/compiler/rustc_trait_selection/src/solve/inspect/analyse.rs
@@ -18,9 +18,9 @@ use rustc_middle::traits::ObligationCause;
 use rustc_middle::traits::solve::{Certainty, Goal, GoalSource, NoSolution, QueryResult};
 use rustc_middle::ty::{TyCtxt, VisitorResult, try_visit};
 use rustc_middle::{bug, ty};
+use rustc_next_trait_solver::canonical::instantiate_canonical_state;
 use rustc_next_trait_solver::resolve::eager_resolve_vars;
-use rustc_next_trait_solver::solve::inspect::{self, instantiate_canonical_state};
-use rustc_next_trait_solver::solve::{MaybeCause, SolverDelegateEvalExt as _};
+use rustc_next_trait_solver::solve::{MaybeCause, SolverDelegateEvalExt as _, inspect};
 use rustc_span::Span;
 use tracing::instrument;
 
diff --git a/compiler/rustc_windows_rc/src/lib.rs b/compiler/rustc_windows_rc/src/lib.rs
index caa5e5ef276..5e95557501e 100644
--- a/compiler/rustc_windows_rc/src/lib.rs
+++ b/compiler/rustc_windows_rc/src/lib.rs
@@ -35,8 +35,11 @@ pub fn compile_windows_resource_file(
     resources_dir.push("resources");
     fs::create_dir_all(&resources_dir).unwrap();
 
-    let resource_compiler =
-        find_resource_compiler(&env::var("CARGO_CFG_TARGET_ARCH").unwrap()).expect("found rc.exe");
+    let resource_compiler = if let Ok(path) = env::var("RUSTC_WINDOWS_RC") {
+        path.into()
+    } else {
+        find_resource_compiler(&env::var("CARGO_CFG_TARGET_ARCH").unwrap()).expect("found rc.exe")
+    };
 
     let rc_path = resources_dir.join(file_stem.with_extension("rc"));