Merge pull request #2349 from rust-lang/rustc-pull

Rustc pull update

7 files changed, 96 insertions, 86 deletions

diff --git a/compiler/rustc_mir_transform/src/early_otherwise_branch.rs b/compiler/rustc_mir_transform/src/early_otherwise_branch.rs
index d49f5d9f9c3..c7feb9e949b 100644
--- a/compiler/rustc_mir_transform/src/early_otherwise_branch.rs
+++ b/compiler/rustc_mir_transform/src/early_otherwise_branch.rs
@@ -223,7 +223,7 @@ impl<'tcx> crate::MirPass<'tcx> for EarlyOtherwiseBranch {
         // Since this optimization adds new basic blocks and invalidates others,
         // clean up the cfg to make it nicer for other passes
         if should_cleanup {
-            simplify_cfg(body);
+            simplify_cfg(tcx, body);
         }
     }
 
diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs
index 69e80ed54ea..c5732194424 100644
--- a/compiler/rustc_mir_transform/src/inline.rs
+++ b/compiler/rustc_mir_transform/src/inline.rs
@@ -63,7 +63,7 @@ impl<'tcx> crate::MirPass<'tcx> for Inline {
         let _guard = span.enter();
         if inline::<NormalInliner<'tcx>>(tcx, body) {
             debug!("running simplify cfg on {:?}", body.source);
-            simplify_cfg(body);
+            simplify_cfg(tcx, body);
             deref_finder(tcx, body);
         }
     }
@@ -99,7 +99,7 @@ impl<'tcx> crate::MirPass<'tcx> for ForceInline {
         let _guard = span.enter();
         if inline::<ForceInliner<'tcx>>(tcx, body) {
             debug!("running simplify cfg on {:?}", body.source);
-            simplify_cfg(body);
+            simplify_cfg(tcx, body);
             deref_finder(tcx, body);
         }
     }
diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs
index 8b4b214a3d4..9732225e48d 100644
--- a/compiler/rustc_mir_transform/src/jump_threading.rs
+++ b/compiler/rustc_mir_transform/src/jump_threading.rs
@@ -90,11 +90,7 @@ impl<'tcx> crate::MirPass<'tcx> for JumpThreading {
         };
 
         for bb in body.basic_blocks.indices() {
-            let old_len = finder.opportunities.len();
-            // If we have any const-eval errors discard any opportunities found
-            if finder.start_from_switch(bb).is_none() {
-                finder.opportunities.truncate(old_len);
-            }
+            finder.start_from_switch(bb);
         }
 
         let opportunities = finder.opportunities;
@@ -201,28 +197,26 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
 
     /// Recursion entry point to find threading opportunities.
     #[instrument(level = "trace", skip(self))]
-    fn start_from_switch(&mut self, bb: BasicBlock) -> Option<()> {
+    fn start_from_switch(&mut self, bb: BasicBlock) {
         let bbdata = &self.body[bb];
         if bbdata.is_cleanup || self.loop_headers.contains(bb) {
-            return Some(());
+            return;
         }
-        let Some((discr, targets)) = bbdata.terminator().kind.as_switch() else { return Some(()) };
-        let Some(discr) = discr.place() else { return Some(()) };
+        let Some((discr, targets)) = bbdata.terminator().kind.as_switch() else { return };
+        let Some(discr) = discr.place() else { return };
         debug!(?discr, ?bb);
 
         let discr_ty = discr.ty(self.body, self.tcx).ty;
-        let Ok(discr_layout) = self.ecx.layout_of(discr_ty) else {
-            return Some(());
-        };
+        let Ok(discr_layout) = self.ecx.layout_of(discr_ty) else { return };
 
-        let Some(discr) = self.map.find(discr.as_ref()) else { return Some(()) };
+        let Some(discr) = self.map.find(discr.as_ref()) else { return };
         debug!(?discr);
 
         let cost = CostChecker::new(self.tcx, self.typing_env, None, self.body);
         let mut state = State::new_reachable();
 
         let conds = if let Some((value, then, else_)) = targets.as_static_if() {
-            let value = ScalarInt::try_from_uint(value, discr_layout.size)?;
+            let Some(value) = ScalarInt::try_from_uint(value, discr_layout.size) else { return };
             self.arena.alloc_from_iter([
                 Condition { value, polarity: Polarity::Eq, target: then },
                 Condition { value, polarity: Polarity::Ne, target: else_ },
@@ -248,10 +242,10 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
         mut state: State<ConditionSet<'a>>,
         mut cost: CostChecker<'_, 'tcx>,
         depth: usize,
-    ) -> Option<()> {
+    ) {
         // Do not thread through loop headers.
         if self.loop_headers.contains(bb) {
-            return Some(());
+            return;
         }
 
         debug!(cost = ?cost.cost());
@@ -259,16 +253,16 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
             self.body.basic_blocks[bb].statements.iter().enumerate().rev()
         {
             if self.is_empty(&state) {
-                return Some(());
+                return;
             }
 
             cost.visit_statement(stmt, Location { block: bb, statement_index });
             if cost.cost() > MAX_COST {
-                return Some(());
+                return;
             }
 
             // Attempt to turn the `current_condition` on `lhs` into a condition on another place.
-            self.process_statement(bb, stmt, &mut state)?;
+            self.process_statement(bb, stmt, &mut state);
 
             // When a statement mutates a place, assignments to that place that happen
             // above the mutation cannot fulfill a condition.
@@ -280,7 +274,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
         }
 
         if self.is_empty(&state) || depth >= MAX_BACKTRACK {
-            return Some(());
+            return;
         }
 
         let last_non_rec = self.opportunities.len();
@@ -293,9 +287,9 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
             match term.kind {
                 TerminatorKind::SwitchInt { ref discr, ref targets } => {
                     self.process_switch_int(discr, targets, bb, &mut state);
-                    self.find_opportunity(pred, state, cost, depth + 1)?;
+                    self.find_opportunity(pred, state, cost, depth + 1);
                 }
-                _ => self.recurse_through_terminator(pred, || state, &cost, depth)?,
+                _ => self.recurse_through_terminator(pred, || state, &cost, depth),
             }
         } else if let &[ref predecessors @ .., last_pred] = &predecessors[..] {
             for &pred in predecessors {
@@ -320,13 +314,12 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
             let first = &mut new_tos[0];
             *first = ThreadingOpportunity { chain: vec![bb], target: first.target };
             self.opportunities.truncate(last_non_rec + 1);
-            return Some(());
+            return;
         }
 
         for op in self.opportunities[last_non_rec..].iter_mut() {
             op.chain.push(bb);
         }
-        Some(())
     }
 
     /// Extract the mutated place from a statement.
@@ -440,23 +433,23 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
         lhs: PlaceIndex,
         rhs: &Operand<'tcx>,
         state: &mut State<ConditionSet<'a>>,
-    ) -> Option<()> {
+    ) {
         match rhs {
             // If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`.
             Operand::Constant(constant) => {
-                let constant = self
-                    .ecx
-                    .eval_mir_constant(&constant.const_, constant.span, None)
-                    .discard_err()?;
+                let Some(constant) =
+                    self.ecx.eval_mir_constant(&constant.const_, constant.span, None).discard_err()
+                else {
+                    return;
+                };
                 self.process_constant(bb, lhs, constant, state);
             }
             // Transfer the conditions on the copied rhs.
             Operand::Move(rhs) | Operand::Copy(rhs) => {
-                let Some(rhs) = self.map.find(rhs.as_ref()) else { return Some(()) };
+                let Some(rhs) = self.map.find(rhs.as_ref()) else { return };
                 state.insert_place_idx(rhs, lhs, &self.map);
             }
         }
-        Some(())
     }
 
     #[instrument(level = "trace", skip(self))]
@@ -466,18 +459,14 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
         lhs_place: &Place<'tcx>,
         rhs: &Rvalue<'tcx>,
         state: &mut State<ConditionSet<'a>>,
-    ) -> Option<()> {
-        let Some(lhs) = self.map.find(lhs_place.as_ref()) else {
-            return Some(());
-        };
+    ) {
+        let Some(lhs) = self.map.find(lhs_place.as_ref()) else { return };
         match rhs {
-            Rvalue::Use(operand) => self.process_operand(bb, lhs, operand, state)?,
+            Rvalue::Use(operand) => self.process_operand(bb, lhs, operand, state),
             // Transfer the conditions on the copy rhs.
-            Rvalue::CopyForDeref(rhs) => {
-                self.process_operand(bb, lhs, &Operand::Copy(*rhs), state)?
-            }
+            Rvalue::CopyForDeref(rhs) => self.process_operand(bb, lhs, &Operand::Copy(*rhs), state),
             Rvalue::Discriminant(rhs) => {
-                let Some(rhs) = self.map.find_discr(rhs.as_ref()) else { return Some(()) };
+                let Some(rhs) = self.map.find_discr(rhs.as_ref()) else { return };
                 state.insert_place_idx(rhs, lhs, &self.map);
             }
             // If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`.
@@ -485,7 +474,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
                 let agg_ty = lhs_place.ty(self.body, self.tcx).ty;
                 let lhs = match kind {
                     // Do not support unions.
-                    AggregateKind::Adt(.., Some(_)) => return Some(()),
+                    AggregateKind::Adt(.., Some(_)) => return,
                     AggregateKind::Adt(_, variant_index, ..) if agg_ty.is_enum() => {
                         if let Some(discr_target) = self.map.apply(lhs, TrackElem::Discriminant)
                             && let Some(discr_value) = self
@@ -498,23 +487,23 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
                         if let Some(idx) = self.map.apply(lhs, TrackElem::Variant(*variant_index)) {
                             idx
                         } else {
-                            return Some(());
+                            return;
                         }
                     }
                     _ => lhs,
                 };
                 for (field_index, operand) in operands.iter_enumerated() {
                     if let Some(field) = self.map.apply(lhs, TrackElem::Field(field_index)) {
-                        self.process_operand(bb, field, operand, state)?;
+                        self.process_operand(bb, field, operand, state);
                     }
                 }
             }
             // Transfer the conditions on the copy rhs, after inverting the value of the condition.
             Rvalue::UnaryOp(UnOp::Not, Operand::Move(place) | Operand::Copy(place)) => {
                 let layout = self.ecx.layout_of(place.ty(self.body, self.tcx).ty).unwrap();
-                let Some(conditions) = state.try_get_idx(lhs, &self.map) else { return Some(()) };
-                let Some(place) = self.map.find(place.as_ref()) else { return Some(()) };
-                let conds = conditions.map(self.arena, |mut cond| {
+                let Some(conditions) = state.try_get_idx(lhs, &self.map) else { return };
+                let Some(place) = self.map.find(place.as_ref()) else { return };
+                let Some(conds) = conditions.map(self.arena, |mut cond| {
                     cond.value = self
                         .ecx
                         .unary_op(UnOp::Not, &ImmTy::from_scalar_int(cond.value, layout))
@@ -522,7 +511,9 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
                         .to_scalar_int()
                         .discard_err()?;
                     Some(cond)
-                })?;
+                }) else {
+                    return;
+                };
                 state.insert_value_idx(place, conds, &self.map);
             }
             // We expect `lhs ?= A`. We found `lhs = Eq(rhs, B)`.
@@ -532,34 +523,38 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
                 box (Operand::Move(place) | Operand::Copy(place), Operand::Constant(value))
                 | box (Operand::Constant(value), Operand::Move(place) | Operand::Copy(place)),
             ) => {
-                let Some(conditions) = state.try_get_idx(lhs, &self.map) else { return Some(()) };
-                let Some(place) = self.map.find(place.as_ref()) else { return Some(()) };
+                let Some(conditions) = state.try_get_idx(lhs, &self.map) else { return };
+                let Some(place) = self.map.find(place.as_ref()) else { return };
                 let equals = match op {
                     BinOp::Eq => ScalarInt::TRUE,
                     BinOp::Ne => ScalarInt::FALSE,
-                    _ => return Some(()),
+                    _ => return,
                 };
                 if value.const_.ty().is_floating_point() {
                     // Floating point equality does not follow bit-patterns.
                     // -0.0 and NaN both have special rules for equality,
                     // and therefore we cannot use integer comparisons for them.
                     // Avoid handling them, though this could be extended in the future.
-                    return Some(());
+                    return;
                 }
-                let value = value.const_.try_eval_scalar_int(self.tcx, self.typing_env)?;
-                let conds = conditions.map(self.arena, |c| {
+                let Some(value) = value.const_.try_eval_scalar_int(self.tcx, self.typing_env)
+                else {
+                    return;
+                };
+                let Some(conds) = conditions.map(self.arena, |c| {
                     Some(Condition {
                         value,
                         polarity: if c.matches(equals) { Polarity::Eq } else { Polarity::Ne },
                         ..c
                     })
-                })?;
+                }) else {
+                    return;
+                };
                 state.insert_value_idx(place, conds, &self.map);
             }
 
             _ => {}
         }
-        Some(())
     }
 
     #[instrument(level = "trace", skip(self))]
@@ -568,7 +563,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
         bb: BasicBlock,
         stmt: &Statement<'tcx>,
         state: &mut State<ConditionSet<'a>>,
-    ) -> Option<()> {
+    ) {
         let register_opportunity = |c: Condition| {
             debug!(?bb, ?c.target, "register");
             self.opportunities.push(ThreadingOpportunity { chain: vec![bb], target: c.target })
@@ -581,32 +576,30 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
             // If we expect `discriminant(place) ?= A`,
             // we have an opportunity if `variant_index ?= A`.
             StatementKind::SetDiscriminant { box place, variant_index } => {
-                let Some(discr_target) = self.map.find_discr(place.as_ref()) else {
-                    return Some(());
-                };
+                let Some(discr_target) = self.map.find_discr(place.as_ref()) else { return };
                 let enum_ty = place.ty(self.body, self.tcx).ty;
                 // `SetDiscriminant` guarantees that the discriminant is now `variant_index`.
                 // Even if the discriminant write does nothing due to niches, it is UB to set the
                 // discriminant when the data does not encode the desired discriminant.
-                let discr =
-                    self.ecx.discriminant_for_variant(enum_ty, *variant_index).discard_err()?;
-                self.process_immediate(bb, discr_target, discr, state);
+                let Some(discr) =
+                    self.ecx.discriminant_for_variant(enum_ty, *variant_index).discard_err()
+                else {
+                    return;
+                };
+                self.process_immediate(bb, discr_target, discr, state)
             }
             // If we expect `lhs ?= true`, we have an opportunity if we assume `lhs == true`.
             StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(
                 Operand::Copy(place) | Operand::Move(place),
             )) => {
-                let Some(conditions) = state.try_get(place.as_ref(), &self.map) else {
-                    return Some(());
-                };
-                conditions.iter_matches(ScalarInt::TRUE).for_each(register_opportunity);
+                let Some(conditions) = state.try_get(place.as_ref(), &self.map) else { return };
+                conditions.iter_matches(ScalarInt::TRUE).for_each(register_opportunity)
             }
             StatementKind::Assign(box (lhs_place, rhs)) => {
-                self.process_assign(bb, lhs_place, rhs, state)?;
+                self.process_assign(bb, lhs_place, rhs, state)
             }
             _ => {}
         }
-        Some(())
     }
 
     #[instrument(level = "trace", skip(self, state, cost))]
@@ -617,7 +610,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
         state: impl FnOnce() -> State<ConditionSet<'a>>,
         cost: &CostChecker<'_, 'tcx>,
         depth: usize,
-    ) -> Option<()> {
+    ) {
         let term = self.body.basic_blocks[bb].terminator();
         let place_to_flood = match term.kind {
             // We come from a target, so those are not possible.
@@ -632,9 +625,9 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> {
             | TerminatorKind::FalseUnwind { .. }
             | TerminatorKind::Yield { .. } => bug!("{term:?} invalid"),
             // Cannot reason about inline asm.
-            TerminatorKind::InlineAsm { .. } => return Some(()),
+            TerminatorKind::InlineAsm { .. } => return,
             // `SwitchInt` is handled specially.
-            TerminatorKind::SwitchInt { .. } => return Some(()),
+            TerminatorKind::SwitchInt { .. } => return,
             // We can recurse, no thing particular to do.
             TerminatorKind::Goto { .. } => None,
             // Flood the overwritten place, and progress through.
diff --git a/compiler/rustc_mir_transform/src/match_branches.rs b/compiler/rustc_mir_transform/src/match_branches.rs
index 5059837328e..b37241185c9 100644
--- a/compiler/rustc_mir_transform/src/match_branches.rs
+++ b/compiler/rustc_mir_transform/src/match_branches.rs
@@ -43,7 +43,7 @@ impl<'tcx> crate::MirPass<'tcx> for MatchBranchSimplification {
         }
 
         if should_cleanup {
-            simplify_cfg(body);
+            simplify_cfg(tcx, body);
         }
     }
 
diff --git a/compiler/rustc_mir_transform/src/remove_place_mention.rs b/compiler/rustc_mir_transform/src/remove_place_mention.rs
index 15fe77d5319..cb598ceb4df 100644
--- a/compiler/rustc_mir_transform/src/remove_place_mention.rs
+++ b/compiler/rustc_mir_transform/src/remove_place_mention.rs
@@ -8,7 +8,7 @@ pub(super) struct RemovePlaceMention;
 
 impl<'tcx> crate::MirPass<'tcx> for RemovePlaceMention {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
-        !sess.opts.unstable_opts.mir_keep_place_mention
+        !sess.opts.unstable_opts.mir_preserve_ub
     }
 
     fn run_pass(&self, _: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
diff --git a/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs b/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs
index 8a8cdafc690..43f80508e4a 100644
--- a/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs
+++ b/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs
@@ -35,7 +35,7 @@ impl<'tcx> crate::MirPass<'tcx> for RemoveUnneededDrops {
         // if we applied optimizations, we potentially have some cfg to cleanup to
         // make it easier for further passes
         if should_simplify {
-            simplify_cfg(body);
+            simplify_cfg(tcx, body);
         }
     }
 
diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs
index 5947637cded..4f2cce8ac10 100644
--- a/compiler/rustc_mir_transform/src/simplify.rs
+++ b/compiler/rustc_mir_transform/src/simplify.rs
@@ -26,6 +26,13 @@
 //! Here the block (`{ return; }`) has the return type `char`, rather than `()`, but the MIR we
 //! naively generate still contains the `_a = ()` write in the unreachable block "after" the
 //! return.
+//!
+//! **WARNING**: This is one of the few optimizations that runs on built and analysis MIR, and
+//! so its effects may affect the type-checking, borrow-checking, and other analysis of MIR.
+//! We must be extremely careful to only apply optimizations that preserve UB and all
+//! non-determinism, since changes here can affect which programs compile in an insta-stable way.
+//! The normal logic that a program with UB can be changed to do anything does not apply to
+//! pre-"runtime" MIR!
 
 use rustc_index::{Idx, IndexSlice, IndexVec};
 use rustc_middle::mir::visit::{MutVisitor, MutatingUseContext, PlaceContext, Visitor};
@@ -66,8 +73,8 @@ impl SimplifyCfg {
     }
 }
 
-pub(super) fn simplify_cfg(body: &mut Body<'_>) {
-    CfgSimplifier::new(body).simplify();
+pub(super) fn simplify_cfg<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    CfgSimplifier::new(tcx, body).simplify();
     remove_dead_blocks(body);
 
     // FIXME: Should probably be moved into some kind of pass manager
@@ -79,9 +86,9 @@ impl<'tcx> crate::MirPass<'tcx> for SimplifyCfg {
         self.name()
     }
 
-    fn run_pass(&self, _: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         debug!("SimplifyCfg({:?}) - simplifying {:?}", self.name(), body.source);
-        simplify_cfg(body);
+        simplify_cfg(tcx, body);
     }
 
     fn is_required(&self) -> bool {
@@ -90,12 +97,13 @@ impl<'tcx> crate::MirPass<'tcx> for SimplifyCfg {
 }
 
 struct CfgSimplifier<'a, 'tcx> {
+    preserve_switch_reads: bool,
     basic_blocks: &'a mut IndexSlice<BasicBlock, BasicBlockData<'tcx>>,
     pred_count: IndexVec<BasicBlock, u32>,
 }
 
 impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> {
-    fn new(body: &'a mut Body<'tcx>) -> Self {
+    fn new(tcx: TyCtxt<'tcx>, body: &'a mut Body<'tcx>) -> Self {
         let mut pred_count = IndexVec::from_elem(0u32, &body.basic_blocks);
 
         // we can't use mir.predecessors() here because that counts
@@ -110,9 +118,12 @@ impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> {
             }
         }
 
+        // Preserve `SwitchInt` reads on built and analysis MIR, or if `-Zmir-preserve-ub`.
+        let preserve_switch_reads = matches!(body.phase, MirPhase::Built | MirPhase::Analysis(_))
+            || tcx.sess.opts.unstable_opts.mir_preserve_ub;
         let basic_blocks = body.basic_blocks_mut();
 
-        CfgSimplifier { basic_blocks, pred_count }
+        CfgSimplifier { preserve_switch_reads, basic_blocks, pred_count }
     }
 
     fn simplify(mut self) {
@@ -253,9 +264,15 @@ impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> {
 
     // turn a branch with all successors identical to a goto
     fn simplify_branch(&mut self, terminator: &mut Terminator<'tcx>) -> bool {
-        match terminator.kind {
-            TerminatorKind::SwitchInt { .. } => {}
-            _ => return false,
+        // Removing a `SwitchInt` terminator may remove reads that result in UB,
+        // so we must not apply this optimization before borrowck or when
+        // `-Zmir-preserve-ub` is set.
+        if self.preserve_switch_reads {
+            return false;
+        }
+
+        let TerminatorKind::SwitchInt { .. } = terminator.kind else {
+            return false;
         };
 
         let first_succ = {