Refactor MIR phases

7 files changed, 90 insertions, 56 deletions

diff --git a/compiler/rustc_mir_transform/src/deaggregator.rs b/compiler/rustc_mir_transform/src/deaggregator.rs
index b93fe5879f4..fe272de20f8 100644
--- a/compiler/rustc_mir_transform/src/deaggregator.rs
+++ b/compiler/rustc_mir_transform/src/deaggregator.rs
@@ -6,10 +6,6 @@ use rustc_middle::ty::TyCtxt;
 pub struct Deaggregator;
 
 impl<'tcx> MirPass<'tcx> for Deaggregator {
-    fn phase_change(&self) -> Option<MirPhase> {
-        Some(MirPhase::Deaggregated)
-    }
-
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         let basic_blocks = body.basic_blocks.as_mut_preserves_cfg();
         for bb in basic_blocks {
diff --git a/compiler/rustc_mir_transform/src/deref_separator.rs b/compiler/rustc_mir_transform/src/deref_separator.rs
index 8869f3f92af..7508df92df1 100644
--- a/compiler/rustc_mir_transform/src/deref_separator.rs
+++ b/compiler/rustc_mir_transform/src/deref_separator.rs
@@ -82,6 +82,5 @@ pub fn deref_finder<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
 impl<'tcx> MirPass<'tcx> for Derefer {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         deref_finder(tcx, body);
-        body.phase = MirPhase::Derefered;
     }
 }
diff --git a/compiler/rustc_mir_transform/src/elaborate_drops.rs b/compiler/rustc_mir_transform/src/elaborate_drops.rs
index 61118ecc8ed..65f4956d23a 100644
--- a/compiler/rustc_mir_transform/src/elaborate_drops.rs
+++ b/compiler/rustc_mir_transform/src/elaborate_drops.rs
@@ -21,10 +21,6 @@ use std::fmt;
 pub struct ElaborateDrops;
 
 impl<'tcx> MirPass<'tcx> for ElaborateDrops {
-    fn phase_change(&self) -> Option<MirPhase> {
-        Some(MirPhase::DropsLowered)
-    }
-
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         debug!("elaborate_drops({:?} @ {:?})", body.source, body.span);
 
diff --git a/compiler/rustc_mir_transform/src/generator.rs b/compiler/rustc_mir_transform/src/generator.rs
index c260611b407..dbff4a6bd69 100644
--- a/compiler/rustc_mir_transform/src/generator.rs
+++ b/compiler/rustc_mir_transform/src/generator.rs
@@ -1240,10 +1240,6 @@ fn create_cases<'tcx>(
 }
 
 impl<'tcx> MirPass<'tcx> for StateTransform {
-    fn phase_change(&self) -> Option<MirPhase> {
-        Some(MirPhase::GeneratorsLowered)
-    }
-
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         let Some(yield_ty) = body.yield_ty() else {
             // This only applies to generators
diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs
index 56fb601d2b5..86ba5bc5128 100644
--- a/compiler/rustc_mir_transform/src/lib.rs
+++ b/compiler/rustc_mir_transform/src/lib.rs
@@ -25,7 +25,9 @@ use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::intravisit::{self, Visitor};
 use rustc_index::vec::IndexVec;
 use rustc_middle::mir::visit::Visitor as _;
-use rustc_middle::mir::{traversal, Body, ConstQualifs, MirPass, MirPhase, Promoted};
+use rustc_middle::mir::{
+    traversal, AnalysisPhase, Body, ConstQualifs, MirPass, MirPhase, Promoted, RuntimePhase,
+};
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, TyCtxt, TypeVisitable};
 
@@ -199,6 +201,8 @@ fn mir_const_qualif(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) ->
 }
 
 /// Make MIR ready for const evaluation. This is run on all MIR, not just on consts!
+/// FIXME(oli-obk): it's unclear whether we still need this phase (and its corresponding query).
+/// We used to have this for pre-miri MIR based const eval.
 fn mir_const<'tcx>(
     tcx: TyCtxt<'tcx>,
     def: ty::WithOptConstParam<LocalDefId>,
@@ -234,7 +238,6 @@ fn mir_const<'tcx>(
             // What we need to do constant evaluation.
             &simplify::SimplifyCfg::new("initial"),
             &rustc_peek::SanityCheck, // Just a lint
-            &marker::PhaseChange(MirPhase::Const),
         ],
     );
     tcx.alloc_steal_mir(body)
@@ -340,7 +343,10 @@ fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -
             pm::run_passes(
                 tcx,
                 &mut body,
-                &[&const_prop::ConstProp, &marker::PhaseChange(MirPhase::Optimized)],
+                &[
+                    &const_prop::ConstProp,
+                    &marker::PhaseChange(MirPhase::Runtime(RuntimePhase::Optimized)),
+                ],
             );
         }
     }
@@ -380,38 +386,61 @@ fn mir_drops_elaborated_and_const_checked<'tcx>(
         body.tainted_by_errors = Some(error_reported);
     }
 
-    // IMPORTANT
-    pm::run_passes(tcx, &mut body, &[&remove_false_edges::RemoveFalseEdges]);
+    run_analysis_to_runtime_passes(tcx, &mut body);
+
+    tcx.alloc_steal_mir(body)
+}
+
+fn run_analysis_to_runtime_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    assert!(body.phase == MirPhase::Analysis(AnalysisPhase::Initial));
+    let did = body.source.def_id();
+
+    debug!("analysis_mir_cleanup({:?})", did);
+    run_analysis_cleanup_passes(tcx, body);
+    assert!(body.phase == MirPhase::Analysis(AnalysisPhase::PostCleanup));
 
     // Do a little drop elaboration before const-checking if `const_precise_live_drops` is enabled.
     if check_consts::post_drop_elaboration::checking_enabled(&ConstCx::new(tcx, &body)) {
         pm::run_passes(
             tcx,
-            &mut body,
+            body,
             &[
-                &simplify::SimplifyCfg::new("remove-false-edges"),
                 &remove_uninit_drops::RemoveUninitDrops,
+                &simplify::SimplifyCfg::new("remove-false-edges"),
             ],
         );
         check_consts::post_drop_elaboration::check_live_drops(tcx, &body); // FIXME: make this a MIR lint
     }
 
-    run_post_borrowck_cleanup_passes(tcx, &mut body);
-    assert!(body.phase == MirPhase::Deaggregated);
-    tcx.alloc_steal_mir(body)
+    debug!("runtime_mir_lowering({:?})", did);
+    run_runtime_lowering_passes(tcx, body);
+    assert!(body.phase == MirPhase::Runtime(RuntimePhase::Initial));
+
+    debug!("runtime_mir_cleanup({:?})", did);
+    run_runtime_cleanup_passes(tcx, body);
+    assert!(body.phase == MirPhase::Runtime(RuntimePhase::PostCleanup));
 }
 
-/// After this series of passes, no lifetime analysis based on borrowing can be done.
-fn run_post_borrowck_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-    debug!("post_borrowck_cleanup({:?})", body.source.def_id());
+// FIXME(JakobDegen): Can we make these lists of passes consts?
 
-    let post_borrowck_cleanup: &[&dyn MirPass<'tcx>] = &[
-        // Remove all things only needed by analysis
+/// After this series of passes, no lifetime analysis based on borrowing can be done.
+fn run_analysis_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    let passes: &[&dyn MirPass<'tcx>] = &[
+        &remove_false_edges::RemoveFalseEdges,
         &simplify_branches::SimplifyConstCondition::new("initial"),
         &remove_noop_landing_pads::RemoveNoopLandingPads,
         &cleanup_post_borrowck::CleanupNonCodegenStatements,
         &simplify::SimplifyCfg::new("early-opt"),
         &deref_separator::Derefer,
+        &marker::PhaseChange(MirPhase::Analysis(AnalysisPhase::PostCleanup)),
+    ];
+
+    pm::run_passes(tcx, body, passes);
+}
+
+/// Returns the sequence of passes that lowers analysis to runtime MIR.
+fn run_runtime_lowering_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    let passes: &[&dyn MirPass<'tcx>] = &[
         // These next passes must be executed together
         &add_call_guards::CriticalCallEdges,
         &elaborate_drops::ElaborateDrops,
@@ -425,16 +454,27 @@ fn run_post_borrowck_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tc
         // `AddRetag` needs to run after `ElaborateDrops`. Otherwise it should run fairly late,
         // but before optimizations begin.
         &elaborate_box_derefs::ElaborateBoxDerefs,
+        &generator::StateTransform,
         &add_retag::AddRetag,
-        &lower_intrinsics::LowerIntrinsics,
-        &simplify::SimplifyCfg::new("elaborate-drops"),
-        // `Deaggregator` is conceptually part of MIR building, some backends rely on it happening
-        // and it can help optimizations.
+        // Deaggregator is necessary for const prop. We may want to consider implementing
+        // CTFE support for aggregates.
         &deaggregator::Deaggregator,
         &Lint(const_prop_lint::ConstProp),
+        &marker::PhaseChange(MirPhase::Runtime(RuntimePhase::Initial)),
+    ];
+    pm::run_passes_no_validate(tcx, body, passes);
+}
+
+/// Returns the sequence of passes that do the initial cleanup of runtime MIR.
+fn run_runtime_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    let passes: &[&dyn MirPass<'tcx>] = &[
+        &elaborate_box_derefs::ElaborateBoxDerefs,
+        &lower_intrinsics::LowerIntrinsics,
+        &simplify::SimplifyCfg::new("elaborate-drops"),
+        &marker::PhaseChange(MirPhase::Runtime(RuntimePhase::PostCleanup)),
     ];
 
-    pm::run_passes(tcx, body, post_borrowck_cleanup);
+    pm::run_passes(tcx, body, passes);
 }
 
 fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
@@ -442,9 +482,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         WithMinOptLevel(1, x)
     }
 
-    // Lowering generator control-flow and variables has to happen before we do anything else
-    // to them. We run some optimizations before that, because they may be harder to do on the state
-    // machine than on MIR with async primitives.
+    // The main optimizations that we do on MIR.
     pm::run_passes(
         tcx,
         body,
@@ -456,17 +494,6 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
             &uninhabited_enum_branching::UninhabitedEnumBranching,
             &o1(simplify::SimplifyCfg::new("after-uninhabited-enum-branching")),
             &inline::Inline,
-            &generator::StateTransform,
-        ],
-    );
-
-    assert!(body.phase == MirPhase::GeneratorsLowered);
-
-    // The main optimizations that we do on MIR.
-    pm::run_passes(
-        tcx,
-        body,
-        &[
             &remove_storage_markers::RemoveStorageMarkers,
             &remove_zsts::RemoveZsts,
             &const_goto::ConstGoto,
@@ -498,7 +525,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
             &deduplicate_blocks::DeduplicateBlocks,
             // Some cleanup necessary at least for LLVM and potentially other codegen backends.
             &add_call_guards::CriticalCallEdges,
-            &marker::PhaseChange(MirPhase::Optimized),
+            &marker::PhaseChange(MirPhase::Runtime(RuntimePhase::Optimized)),
             // Dump the end result for testing and debugging purposes.
             &dump_mir::Marker("PreCodegen"),
         ],
@@ -557,7 +584,7 @@ fn promoted_mir<'tcx>(
         if let Some(error_reported) = tainted_by_errors {
             body.tainted_by_errors = Some(error_reported);
         }
-        run_post_borrowck_cleanup_passes(tcx, body);
+        run_analysis_to_runtime_passes(tcx, body);
     }
 
     debug_assert!(!promoted.has_free_regions(), "Free regions in promoted MIR");
diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs
index e27d4ab1688..0b21d281524 100644
--- a/compiler/rustc_mir_transform/src/pass_manager.rs
+++ b/compiler/rustc_mir_transform/src/pass_manager.rs
@@ -1,6 +1,6 @@
 use std::borrow::Cow;
 
-use rustc_middle::mir::{self, Body, MirPhase};
+use rustc_middle::mir::{self, Body, MirPhase, RuntimePhase};
 use rustc_middle::ty::TyCtxt;
 use rustc_session::Session;
 
@@ -72,11 +72,30 @@ where
     }
 }
 
+/// Run the sequence of passes without validating the MIR after each pass. The MIR is still
+/// validated at the end.
+pub fn run_passes_no_validate<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    body: &mut Body<'tcx>,
+    passes: &[&dyn MirPass<'tcx>],
+) {
+    run_passes_inner(tcx, body, passes, false);
+}
+
 pub fn run_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, passes: &[&dyn MirPass<'tcx>]) {
+    run_passes_inner(tcx, body, passes, true);
+}
+
+fn run_passes_inner<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    body: &mut Body<'tcx>,
+    passes: &[&dyn MirPass<'tcx>],
+    validate_each: bool,
+) {
     let start_phase = body.phase;
     let mut cnt = 0;
 
-    let validate = tcx.sess.opts.unstable_opts.validate_mir;
+    let validate = validate_each & tcx.sess.opts.unstable_opts.validate_mir;
     let overridden_passes = &tcx.sess.opts.unstable_opts.mir_enable_passes;
     trace!(?overridden_passes);
 
@@ -127,7 +146,7 @@ pub fn run_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, passes: &[&dyn
         }
     }
 
-    if validate || body.phase == MirPhase::Optimized {
+    if validate || body.phase == MirPhase::Runtime(RuntimePhase::Optimized) {
         validate_body(tcx, body, format!("end of phase transition to {:?}", body.phase));
     }
 }
@@ -144,7 +163,7 @@ pub fn dump_mir<'tcx>(
     cnt: usize,
     is_after: bool,
 ) {
-    let phase_index = phase as u32;
+    let phase_index = phase.phase_index();
 
     mir::dump_mir(
         tcx,
diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs
index 3620e94bec7..2b21fbe0a61 100644
--- a/compiler/rustc_mir_transform/src/shim.rs
+++ b/compiler/rustc_mir_transform/src/shim.rs
@@ -17,8 +17,8 @@ use std::iter;
 
 use crate::util::expand_aggregate;
 use crate::{
-    abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, marker, pass_manager as pm,
-    remove_noop_landing_pads, simplify,
+    abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, deref_separator, marker,
+    pass_manager as pm, remove_noop_landing_pads, simplify,
 };
 use rustc_middle::mir::patch::MirPatch;
 use rustc_mir_dataflow::elaborate_drops::{self, DropElaborator, DropFlagMode, DropStyle};
@@ -92,11 +92,12 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<'
         &mut result,
         &[
             &add_moves_for_packed_drops::AddMovesForPackedDrops,
+            &deref_separator::Derefer,
             &remove_noop_landing_pads::RemoveNoopLandingPads,
             &simplify::SimplifyCfg::new("make_shim"),
             &add_call_guards::CriticalCallEdges,
             &abort_unwinding_calls::AbortUnwindingCalls,
-            &marker::PhaseChange(MirPhase::Const),
+            &marker::PhaseChange(MirPhase::Runtime(RuntimePhase::Optimized)),
         ],
     );