replace location-insensitive analysis with location-sensitive analysis

we're in in the endgame now

set up the location-sensitive analysis end to end:
- stop recording inflowing loans and loan liveness in liveness
- replace location-insensitive liveness data with live loans computed by
  reachability
- remove equivalence between polonius scopes and NLL scopes, and only
  run one scope computation

6 files changed, 59 insertions, 140 deletions

diff --git a/compiler/rustc_borrowck/src/dataflow.rs b/compiler/rustc_borrowck/src/dataflow.rs
index a7a6f2da509..3a200642ce5 100644
--- a/compiler/rustc_borrowck/src/dataflow.rs
+++ b/compiler/rustc_borrowck/src/dataflow.rs
@@ -325,10 +325,17 @@ impl<'tcx> PoloniusOutOfScopePrecomputer<'_, 'tcx> {
         let sccs = self.regioncx.constraint_sccs();
         let universal_regions = self.regioncx.universal_regions();
 
+        // The loop below was useful for the location-insensitive analysis but shouldn't be
+        // impactful in the location-sensitive case. It seems that it does, however, as without it a
+        // handful of tests fail. That likely means some liveness or outlives data related to choice
+        // regions is missing
+        // FIXME: investigate the impact of loans traversing applied member constraints and why some
+        // tests fail otherwise.
+        //
         // We first handle the cases where the loan doesn't go out of scope, depending on the
         // issuing region's successors.
         for successor in graph::depth_first_search(&self.regioncx.region_graph(), issuing_region) {
-            // 1. Via applied member constraints
+            // Via applied member constraints
             //
             // The issuing region can flow into the choice regions, and they are either:
             // - placeholders or free regions themselves,
@@ -346,14 +353,6 @@ impl<'tcx> PoloniusOutOfScopePrecomputer<'_, 'tcx> {
                     return;
                 }
             }
-
-            // 2. Via regions that are live at all points: placeholders and free regions.
-            //
-            // If the issuing region outlives such a region, its loan escapes the function and
-            // cannot go out of scope. We can early return.
-            if self.regioncx.is_region_live_at_all_points(successor) {
-                return;
-            }
         }
 
         let first_block = loan_issued_at.block;
@@ -461,34 +460,26 @@ impl<'a, 'tcx> Borrows<'a, 'tcx> {
         regioncx: &RegionInferenceContext<'tcx>,
         borrow_set: &'a BorrowSet<'tcx>,
     ) -> Self {
-        let mut borrows_out_of_scope_at_location =
-            calculate_borrows_out_of_scope_at_location(body, regioncx, borrow_set);
-
-        // The in-tree polonius analysis computes loans going out of scope using the set-of-loans
-        // model, and makes sure they're identical to the existing computation of the set-of-points
-        // model.
-        if tcx.sess.opts.unstable_opts.polonius.is_next_enabled() {
-            let mut polonius_prec = PoloniusOutOfScopePrecomputer::new(body, regioncx);
-            for (loan_idx, loan_data) in borrow_set.iter_enumerated() {
-                let issuing_region = loan_data.region;
-                let loan_issued_at = loan_data.reserve_location;
-
-                polonius_prec.precompute_loans_out_of_scope(
-                    loan_idx,
-                    issuing_region,
-                    loan_issued_at,
-                );
-            }
-
-            assert_eq!(
-                borrows_out_of_scope_at_location, polonius_prec.loans_out_of_scope_at_location,
-                "polonius loan scopes differ from NLL borrow scopes, for body {:?}",
-                body.span,
-            );
-
-            borrows_out_of_scope_at_location = polonius_prec.loans_out_of_scope_at_location;
-        }
+        let borrows_out_of_scope_at_location =
+            if !tcx.sess.opts.unstable_opts.polonius.is_next_enabled() {
+                calculate_borrows_out_of_scope_at_location(body, regioncx, borrow_set)
+            } else {
+                // The in-tree polonius analysis computes loans going out of scope using the
+                // set-of-loans model.
+                let mut polonius_prec = PoloniusOutOfScopePrecomputer::new(body, regioncx);
+                for (loan_idx, loan_data) in borrow_set.iter_enumerated() {
+                    let issuing_region = loan_data.region;
+                    let loan_issued_at = loan_data.reserve_location;
+
+                    polonius_prec.precompute_loans_out_of_scope(
+                        loan_idx,
+                        issuing_region,
+                        loan_issued_at,
+                    );
+                }
 
+                polonius_prec.loans_out_of_scope_at_location
+            };
         Borrows { tcx, body, borrow_set, borrows_out_of_scope_at_location }
     }
 
diff --git a/compiler/rustc_borrowck/src/nll.rs b/compiler/rustc_borrowck/src/nll.rs
index 590cbef05d5..35264bd1a70 100644
--- a/compiler/rustc_borrowck/src/nll.rs
+++ b/compiler/rustc_borrowck/src/nll.rs
@@ -145,7 +145,7 @@ pub(crate) fn compute_regions<'a, 'tcx>(
     // If requested for `-Zpolonius=next`, convert NLL constraints to localized outlives constraints
     // and use them to compute loan liveness.
     let localized_outlives_constraints = polonius_context.as_ref().map(|polonius_context| {
-        polonius_context.compute_loan_liveness(infcx.tcx, &regioncx, body, borrow_set)
+        polonius_context.compute_loan_liveness(infcx.tcx, &mut regioncx, body, borrow_set)
     });
 
     // If requested: dump NLL facts, and run legacy polonius analysis.
diff --git a/compiler/rustc_borrowck/src/polonius/mod.rs b/compiler/rustc_borrowck/src/polonius/mod.rs
index 313ed5d6649..52a5f75d8a2 100644
--- a/compiler/rustc_borrowck/src/polonius/mod.rs
+++ b/compiler/rustc_borrowck/src/polonius/mod.rs
@@ -98,7 +98,7 @@ impl PoloniusContext {
     pub(crate) fn compute_loan_liveness<'tcx>(
         &self,
         tcx: TyCtxt<'tcx>,
-        regioncx: &RegionInferenceContext<'tcx>,
+        regioncx: &mut RegionInferenceContext<'tcx>,
         body: &Body<'tcx>,
         borrow_set: &BorrowSet<'tcx>,
     ) -> LocalizedOutlivesConstraintSet {
@@ -126,15 +126,14 @@ impl PoloniusContext {
 
         // Now that we have a complete graph, we can compute reachability to trace the liveness of
         // loans for the next step in the chain, the NLL loan scope and active loans computations.
-        let _live_loans = compute_loan_liveness(
+        let live_loans = compute_loan_liveness(
             tcx,
             body,
             regioncx.liveness_constraints(),
             borrow_set,
             &localized_outlives_constraints,
         );
-        // FIXME: record the live loans in the regioncx's liveness constraints, where the
-        // location-insensitive variant's data is stored.
+        regioncx.record_live_loans(live_loans);
 
         localized_outlives_constraints
     }
diff --git a/compiler/rustc_borrowck/src/region_infer/mod.rs b/compiler/rustc_borrowck/src/region_infer/mod.rs
index c177538ee17..d2268c4779d 100644
--- a/compiler/rustc_borrowck/src/region_infer/mod.rs
+++ b/compiler/rustc_borrowck/src/region_infer/mod.rs
@@ -31,6 +31,7 @@ use crate::constraints::{ConstraintSccIndex, OutlivesConstraint, OutlivesConstra
 use crate::dataflow::BorrowIndex;
 use crate::diagnostics::{RegionErrorKind, RegionErrors, UniverseInfo};
 use crate::member_constraints::{MemberConstraintSet, NllMemberConstraintIndex};
+use crate::polonius::LiveLoans;
 use crate::polonius::legacy::PoloniusOutput;
 use crate::region_infer::reverse_sccs::ReverseSccGraph;
 use crate::region_infer::values::{LivenessValues, RegionElement, RegionValues, ToElementIndex};
@@ -2171,28 +2172,6 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         self.constraint_graph.region_graph(&self.constraints, self.universal_regions().fr_static)
     }
 
-    /// Returns whether the given region is considered live at all points: whether it is a
-    /// placeholder or a free region.
-    pub(crate) fn is_region_live_at_all_points(&self, region: RegionVid) -> bool {
-        // FIXME: there must be a cleaner way to find this information. At least, when
-        // higher-ranked subtyping is abstracted away from the borrowck main path, we'll only
-        // need to check whether this is a universal region.
-        let origin = self.region_definition(region).origin;
-        let live_at_all_points = matches!(
-            origin,
-            NllRegionVariableOrigin::Placeholder(_) | NllRegionVariableOrigin::FreeRegion
-        );
-        live_at_all_points
-    }
-
-    /// Returns whether the `loan_idx` is live at the given `location`: whether its issuing
-    /// region is contained within the type of a variable that is live at this point.
-    /// Note: for now, the sets of live loans is only available when using `-Zpolonius=next`.
-    pub(crate) fn is_loan_live_at(&self, loan_idx: BorrowIndex, location: Location) -> bool {
-        let point = self.liveness_constraints.point_from_location(location);
-        self.liveness_constraints.is_loan_live_at(loan_idx, point)
-    }
-
     /// Returns the representative `RegionVid` for a given SCC.
     /// See `RegionTracker` for how a region variable ID is chosen.
     ///
@@ -2208,6 +2187,20 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     pub(crate) fn liveness_constraints(&self) -> &LivenessValues {
         &self.liveness_constraints
     }
+
+    /// When using `-Zpolonius=next`, records the given live loans for the loan scopes and active
+    /// loans dataflow computations.
+    pub(crate) fn record_live_loans(&mut self, live_loans: LiveLoans) {
+        self.liveness_constraints.record_live_loans(live_loans);
+    }
+
+    /// Returns whether the `loan_idx` is live at the given `location`: whether its issuing
+    /// region is contained within the type of a variable that is live at this point.
+    /// Note: for now, the sets of live loans is only available when using `-Zpolonius=next`.
+    pub(crate) fn is_loan_live_at(&self, loan_idx: BorrowIndex, location: Location) -> bool {
+        let point = self.liveness_constraints.point_from_location(location);
+        self.liveness_constraints.is_loan_live_at(loan_idx, point)
+    }
 }
 
 impl<'tcx> RegionDefinition<'tcx> {
diff --git a/compiler/rustc_borrowck/src/region_infer/values.rs b/compiler/rustc_borrowck/src/region_infer/values.rs
index 11fb125ca22..f1bcb353dc6 100644
--- a/compiler/rustc_borrowck/src/region_infer/values.rs
+++ b/compiler/rustc_borrowck/src/region_infer/values.rs
@@ -11,6 +11,7 @@ use rustc_mir_dataflow::points::{DenseLocationMap, PointIndex};
 use tracing::debug;
 
 use crate::BorrowIndex;
+use crate::polonius::LiveLoans;
 
 rustc_index::newtype_index! {
     /// A single integer representing a `ty::Placeholder`.
@@ -50,29 +51,8 @@ pub(crate) struct LivenessValues {
     /// region is live, only that it is.
     points: Option<SparseIntervalMatrix<RegionVid, PointIndex>>,
 
-    /// When using `-Zpolonius=next`, for each point: the loans flowing into the live regions at
-    /// that point.
-    pub(crate) loans: Option<LiveLoans>,
-}
-
-/// Data used to compute the loans that are live at a given point in the CFG, when using
-/// `-Zpolonius=next`.
-pub(crate) struct LiveLoans {
-    /// The set of loans that flow into a given region. When individual regions are marked as live
-    /// in the CFG, these inflowing loans are recorded as live.
-    pub(crate) inflowing_loans: SparseBitMatrix<RegionVid, BorrowIndex>,
-
-    /// The set of loans that are live at a given point in the CFG.
-    pub(crate) live_loans: SparseBitMatrix<PointIndex, BorrowIndex>,
-}
-
-impl LiveLoans {
-    pub(crate) fn new(num_loans: usize) -> Self {
-        LiveLoans {
-            live_loans: SparseBitMatrix::new(num_loans),
-            inflowing_loans: SparseBitMatrix::new(num_loans),
-        }
-    }
+    /// When using `-Zpolonius=next`, the set of loans that are live at a given point in the CFG.
+    live_loans: Option<LiveLoans>,
 }
 
 impl LivenessValues {
@@ -82,7 +62,7 @@ impl LivenessValues {
             live_regions: None,
             points: Some(SparseIntervalMatrix::new(location_map.num_points())),
             location_map,
-            loans: None,
+            live_loans: None,
         }
     }
 
@@ -95,7 +75,7 @@ impl LivenessValues {
             live_regions: Some(Default::default()),
             points: None,
             location_map,
-            loans: None,
+            live_loans: None,
         }
     }
 
@@ -129,13 +109,6 @@ impl LivenessValues {
         } else if self.location_map.point_in_range(point) {
             self.live_regions.as_mut().unwrap().insert(region);
         }
-
-        // When available, record the loans flowing into this region as live at the given point.
-        if let Some(loans) = self.loans.as_mut() {
-            if let Some(inflowing) = loans.inflowing_loans.row(region) {
-                loans.live_loans.union_row(point, inflowing);
-            }
-        }
     }
 
     /// Records `region` as being live at all the given `points`.
@@ -146,17 +119,6 @@ impl LivenessValues {
         } else if points.iter().any(|point| self.location_map.point_in_range(point)) {
             self.live_regions.as_mut().unwrap().insert(region);
         }
-
-        // When available, record the loans flowing into this region as live at the given points.
-        if let Some(loans) = self.loans.as_mut() {
-            if let Some(inflowing) = loans.inflowing_loans.row(region) {
-                if !inflowing.is_empty() {
-                    for point in points.iter() {
-                        loans.live_loans.union_row(point, inflowing);
-                    }
-                }
-            }
-        }
     }
 
     /// Records `region` as being live at all the control-flow points.
@@ -213,12 +175,17 @@ impl LivenessValues {
         self.location_map.to_location(point)
     }
 
+    /// When using `-Zpolonius=next`, records the given live loans for the loan scopes and active
+    /// loans dataflow computations.
+    pub(crate) fn record_live_loans(&mut self, live_loans: LiveLoans) {
+        self.live_loans = Some(live_loans);
+    }
+
     /// When using `-Zpolonius=next`, returns whether the `loan_idx` is live at the given `point`.
     pub(crate) fn is_loan_live_at(&self, loan_idx: BorrowIndex, point: PointIndex) -> bool {
-        self.loans
+        self.live_loans
             .as_ref()
             .expect("Accessing live loans requires `-Zpolonius=next`")
-            .live_loans
             .contains(point, loan_idx)
     }
 }
diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
index 4c0d3138f2d..c564d85616e 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
@@ -16,7 +16,7 @@ use rustc_trait_selection::traits::query::type_op::{DropckOutlives, TypeOp, Type
 use tracing::debug;
 
 use crate::polonius;
-use crate::region_infer::values::{self, LiveLoans};
+use crate::region_infer::values;
 use crate::type_check::liveness::local_use_map::LocalUseMap;
 use crate::type_check::{NormalizeLocation, TypeChecker};
 
@@ -44,37 +44,6 @@ pub(super) fn trace<'a, 'tcx>(
     boring_locals: Vec<Local>,
 ) {
     let local_use_map = &LocalUseMap::build(&relevant_live_locals, location_map, body);
-
-    // When using `-Zpolonius=next`, compute the set of loans that can reach a given region.
-    if typeck.tcx().sess.opts.unstable_opts.polonius.is_next_enabled() {
-        let borrow_set = &typeck.borrow_set;
-        let mut live_loans = LiveLoans::new(borrow_set.len());
-        let outlives_constraints = &typeck.constraints.outlives_constraints;
-        let graph = outlives_constraints.graph(typeck.infcx.num_region_vars());
-        let region_graph =
-            graph.region_graph(outlives_constraints, typeck.universal_regions.fr_static);
-
-        // Traverse each issuing region's constraints, and record the loan as flowing into the
-        // outlived region.
-        for (loan, issuing_region_data) in borrow_set.iter_enumerated() {
-            for succ in rustc_data_structures::graph::depth_first_search(
-                &region_graph,
-                issuing_region_data.region,
-            ) {
-                // We don't need to mention that a loan flows into its issuing region.
-                if succ == issuing_region_data.region {
-                    continue;
-                }
-
-                live_loans.inflowing_loans.insert(succ, loan);
-            }
-        }
-
-        // Store the inflowing loans in the liveness constraints: they will be used to compute live
-        // loans when liveness data is recorded there.
-        typeck.constraints.liveness_constraints.loans = Some(live_loans);
-    };
-
     let cx = LivenessContext {
         typeck,
         body,