Rollup merge of #110614 - compiler-errors:new-solver-overflow-response, r=lcnr

Clear response values for overflow in new solver

When we have an overflow, return a trivial query response. This fixes an ICE with the code described in #110544:

```rust
trait Trait {}

struct W<T>(T);

impl<T, U> Trait for W<(W<T>, W<U>)>
where
    W<T>: Trait,
    W<U>: Trait,
{}

fn impls<T: Trait>() {}

fn main() {
    impls::<W<_>>()
}
```

Where, while proving `W<?0>: Trait`, we overflow but still apply the query response of `?0 = (W<?1>, W<?2>)`. Then while re-processing the query to validate that our evaluation result was stable, we get a different query response that looks like `?1 = (W<?3>, W<?4>), ?2 = (W<?5>, W<?6>)`, and so we trigger the ICE.

Also, by returning a trivial query response we also avoid the infinite-loop/OOM behavior of the old solver.

r? ``@lcnr``

3 files changed, 79 insertions, 20 deletions

diff --git a/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs b/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs
index bd52957d162..63a73f8d50d 100644
--- a/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs
+++ b/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs
@@ -3,7 +3,8 @@ use rustc_infer::infer::at::ToTrace;
 use rustc_infer::infer::canonical::CanonicalVarValues;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
 use rustc_infer::infer::{
-    DefineOpaqueTypes, InferCtxt, InferOk, LateBoundRegionConversionTime, TyCtxtInferExt,
+    DefineOpaqueTypes, InferCtxt, InferOk, LateBoundRegionConversionTime, RegionVariableOrigin,
+    TyCtxtInferExt,
 };
 use rustc_infer::traits::query::NoSolution;
 use rustc_infer::traits::ObligationCause;
@@ -223,18 +224,20 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
         {
             debug!("rerunning goal to check result is stable");
             let (_orig_values, canonical_goal) = self.canonicalize_goal(goal);
-            let canonical_response =
+            let new_canonical_response =
                 EvalCtxt::evaluate_canonical_goal(self.tcx(), self.search_graph, canonical_goal)?;
-            if !canonical_response.value.var_values.is_identity() {
+            if !new_canonical_response.value.var_values.is_identity() {
                 bug!(
                     "unstable result: re-canonicalized goal={canonical_goal:#?} \
-                     response={canonical_response:#?}"
+                    first_response={canonical_response:#?} \
+                    second_response={new_canonical_response:#?}"
                 );
             }
-            if certainty != canonical_response.value.certainty {
+            if certainty != new_canonical_response.value.certainty {
                 bug!(
                     "unstable certainty: {certainty:#?} re-canonicalized goal={canonical_goal:#?} \
-                     response={canonical_response:#?}"
+                     first_response={canonical_response:#?} \
+                     second_response={new_canonical_response:#?}"
                 );
             }
         }
@@ -434,6 +437,10 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         })
     }
 
+    pub(super) fn next_region_infer(&self) -> ty::Region<'tcx> {
+        self.infcx.next_region_var(RegionVariableOrigin::MiscVariable(DUMMY_SP))
+    }
+
     pub(super) fn next_const_infer(&self, ty: Ty<'tcx>) -> ty::Const<'tcx> {
         self.infcx.next_const_var(
             ty,
diff --git a/compiler/rustc_trait_selection/src/solve/eval_ctxt/canonical.rs b/compiler/rustc_trait_selection/src/solve/eval_ctxt/canonical.rs
index 226d29687e3..67ad7fb4bd2 100644
--- a/compiler/rustc_trait_selection/src/solve/eval_ctxt/canonical.rs
+++ b/compiler/rustc_trait_selection/src/solve/eval_ctxt/canonical.rs
@@ -16,7 +16,7 @@ use rustc_infer::infer::canonical::query_response::make_query_region_constraints
 use rustc_infer::infer::canonical::CanonicalVarValues;
 use rustc_infer::infer::canonical::{CanonicalExt, QueryRegionConstraints};
 use rustc_middle::traits::query::NoSolution;
-use rustc_middle::traits::solve::{ExternalConstraints, ExternalConstraintsData};
+use rustc_middle::traits::solve::{ExternalConstraints, ExternalConstraintsData, MaybeCause};
 use rustc_middle::ty::{self, BoundVar, GenericArgKind};
 use rustc_span::DUMMY_SP;
 use std::iter;
@@ -60,9 +60,27 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
 
         let certainty = certainty.unify_with(goals_certainty);
 
-        let external_constraints = self.compute_external_query_constraints()?;
+        let response = match certainty {
+            Certainty::Yes | Certainty::Maybe(MaybeCause::Ambiguity) => {
+                let external_constraints = self.compute_external_query_constraints()?;
+                Response { var_values: self.var_values, external_constraints, certainty }
+            }
+            Certainty::Maybe(MaybeCause::Overflow) => {
+                // If we have overflow, it's probable that we're substituting a type
+                // into itself infinitely and any partial substitutions in the query
+                // response are probably not useful anyways, so just return an empty
+                // query response.
+                //
+                // This may prevent us from potentially useful inference, e.g.
+                // 2 candidates, one ambiguous and one overflow, which both
+                // have the same inference constraints.
+                //
+                // Changing this to retain some constraints in the future
+                // won't be a breaking change, so this is good enough for now.
+                return Ok(self.make_ambiguous_response_no_constraints(MaybeCause::Overflow));
+            }
+        };
 
-        let response = Response { var_values: self.var_values, external_constraints, certainty };
         let canonical = Canonicalizer::canonicalize(
             self.infcx,
             CanonicalizeMode::Response { max_input_universe: self.max_input_universe },
@@ -72,6 +90,40 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         Ok(canonical)
     }
 
+    /// Constructs a totally unconstrained, ambiguous response to a goal.
+    ///
+    /// Take care when using this, since often it's useful to respond with
+    /// ambiguity but return constrained variables to guide inference.
+    pub(in crate::solve) fn make_ambiguous_response_no_constraints(
+        &self,
+        maybe_cause: MaybeCause,
+    ) -> CanonicalResponse<'tcx> {
+        let unconstrained_response = Response {
+            var_values: CanonicalVarValues {
+                var_values: self.tcx().mk_substs_from_iter(self.var_values.var_values.iter().map(
+                    |arg| -> ty::GenericArg<'tcx> {
+                        match arg.unpack() {
+                            GenericArgKind::Lifetime(_) => self.next_region_infer().into(),
+                            GenericArgKind::Type(_) => self.next_ty_infer().into(),
+                            GenericArgKind::Const(ct) => self.next_const_infer(ct.ty()).into(),
+                        }
+                    },
+                )),
+            },
+            external_constraints: self
+                .tcx()
+                .mk_external_constraints(ExternalConstraintsData::default()),
+            certainty: Certainty::Maybe(maybe_cause),
+        };
+
+        Canonicalizer::canonicalize(
+            self.infcx,
+            CanonicalizeMode::Response { max_input_universe: self.max_input_universe },
+            &mut Default::default(),
+            unconstrained_response,
+        )
+    }
+
     #[instrument(level = "debug", skip(self), ret)]
     fn compute_external_query_constraints(&self) -> Result<ExternalConstraints<'tcx>, NoSolution> {
         // Cannot use `take_registered_region_obligations` as we may compute the response
diff --git a/compiler/rustc_trait_selection/src/solve/mod.rs b/compiler/rustc_trait_selection/src/solve/mod.rs
index 19bcbd46144..d94679fef28 100644
--- a/compiler/rustc_trait_selection/src/solve/mod.rs
+++ b/compiler/rustc_trait_selection/src/solve/mod.rs
@@ -340,17 +340,17 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         if responses.is_empty() {
             return Err(NoSolution);
         }
-        let certainty = responses.iter().fold(Certainty::AMBIGUOUS, |certainty, response| {
-            certainty.unify_with(response.value.certainty)
-        });
-
-        let response = self.evaluate_added_goals_and_make_canonical_response(certainty);
-        if let Ok(response) = response {
-            assert!(response.has_no_inference_or_external_constraints());
-            Ok(response)
-        } else {
-            bug!("failed to make floundered response: {responses:?}");
-        }
+
+        let Certainty::Maybe(maybe_cause) = responses.iter().fold(
+            Certainty::AMBIGUOUS,
+            |certainty, response| {
+                certainty.unify_with(response.value.certainty)
+            },
+        ) else {
+            bug!("expected flounder response to be ambiguous")
+        };
+
+        Ok(self.make_ambiguous_response_no_constraints(maybe_cause))
     }
 }