Auto merge of #119767 - GuillaumeGomez:rollup-fbp26yb, r=GuillaumeGomez

Rollup of 9 pull requests

Successful merges:

 - #117556 (Disallow reference to `static mut` and adding `static_mut_ref` lint)
 - #118748 (std: getrandom simplification for freebsd.)
 - #119282 (Rework and improve the unstable documentation of check-cfg)
 - #119527 (don't reexport atomic::ordering via rustc_data_structures, use std import)
 - #119668 (Simplify implementation of MIR promotion)
 - #119699 (Merge dead bb pruning and unreachable bb deduplication.)
 - #119723 (Remove `-Zdont-buffer-diagnostics`.)
 - #119756 (rustdoc-search: reuse individual types in function signatures)
 - #119758 (GNU/Hurd: unconditionally use inline stack probes)

r? `@ghost`
`@rustbot` modify labels: rollup

10 files changed, 1000 insertions, 66 deletions

diff --git a/compiler/rustc_mir_transform/src/const_goto.rs b/compiler/rustc_mir_transform/src/const_goto.rs
index 3884346076e..cb5b66b314d 100644
--- a/compiler/rustc_mir_transform/src/const_goto.rs
+++ b/compiler/rustc_mir_transform/src/const_goto.rs
@@ -51,7 +51,7 @@ impl<'tcx> MirPass<'tcx> for ConstGoto {
         // if we applied optimizations, we potentially have some cfg to cleanup to
         // make it easier for further passes
         if should_simplify {
-            simplify_cfg(tcx, body);
+            simplify_cfg(body);
             simplify_locals(body, tcx);
         }
     }
diff --git a/compiler/rustc_mir_transform/src/deduplicate_blocks.rs b/compiler/rustc_mir_transform/src/deduplicate_blocks.rs
index b40b2ec8bfd..824974970bb 100644
--- a/compiler/rustc_mir_transform/src/deduplicate_blocks.rs
+++ b/compiler/rustc_mir_transform/src/deduplicate_blocks.rs
@@ -25,7 +25,7 @@ impl<'tcx> MirPass<'tcx> for DeduplicateBlocks {
         if has_opts_to_apply {
             let mut opt_applier = OptApplier { tcx, duplicates };
             opt_applier.visit_body(body);
-            simplify_cfg(tcx, body);
+            simplify_cfg(body);
         }
     }
 }
diff --git a/compiler/rustc_mir_transform/src/early_otherwise_branch.rs b/compiler/rustc_mir_transform/src/early_otherwise_branch.rs
index 6eb6cb069fe..0d600f0f937 100644
--- a/compiler/rustc_mir_transform/src/early_otherwise_branch.rs
+++ b/compiler/rustc_mir_transform/src/early_otherwise_branch.rs
@@ -212,7 +212,7 @@ impl<'tcx> MirPass<'tcx> for EarlyOtherwiseBranch {
         // Since this optimization adds new basic blocks and invalidates others,
         // clean up the cfg to make it nicer for other passes
         if should_cleanup {
-            simplify_cfg(tcx, body);
+            simplify_cfg(body);
         }
     }
 }
diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs
index 8ad804bf3e7..ebefc3b47cc 100644
--- a/compiler/rustc_mir_transform/src/inline.rs
+++ b/compiler/rustc_mir_transform/src/inline.rs
@@ -15,7 +15,7 @@ use rustc_target::abi::FieldIdx;
 use rustc_target::spec::abi::Abi;
 
 use crate::cost_checker::CostChecker;
-use crate::simplify::{remove_dead_blocks, CfgSimplifier};
+use crate::simplify::simplify_cfg;
 use crate::util;
 use std::iter;
 use std::ops::{Range, RangeFrom};
@@ -56,8 +56,7 @@ impl<'tcx> MirPass<'tcx> for Inline {
         let _guard = span.enter();
         if inline(tcx, body) {
             debug!("running simplify cfg on {:?}", body.source);
-            CfgSimplifier::new(body).simplify();
-            remove_dead_blocks(body);
+            simplify_cfg(body);
             deref_finder(tcx, body);
         }
     }
diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs
index 2c1602dadc1..43c5b3873b9 100644
--- a/compiler/rustc_mir_transform/src/lib.rs
+++ b/compiler/rustc_mir_transform/src/lib.rs
@@ -1,5 +1,6 @@
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
+#![feature(assert_matches)]
 #![feature(box_patterns)]
 #![feature(cow_is_borrowed)]
 #![feature(decl_macro)]
@@ -94,6 +95,7 @@ mod multiple_return_terminators;
 mod normalize_array_len;
 mod nrvo;
 mod prettify;
+mod promote_consts;
 mod ref_prop;
 mod remove_noop_landing_pads;
 mod remove_storage_markers;
@@ -115,7 +117,6 @@ mod uninhabited_enum_branching;
 mod unreachable_prop;
 
 use rustc_const_eval::transform::check_consts::{self, ConstCx};
-use rustc_const_eval::transform::promote_consts;
 use rustc_const_eval::transform::validate;
 use rustc_mir_dataflow::rustc_peek;
 
diff --git a/compiler/rustc_mir_transform/src/match_branches.rs b/compiler/rustc_mir_transform/src/match_branches.rs
index 1c4aa37d57f..6d4332793af 100644
--- a/compiler/rustc_mir_transform/src/match_branches.rs
+++ b/compiler/rustc_mir_transform/src/match_branches.rs
@@ -174,7 +174,7 @@ impl<'tcx> MirPass<'tcx> for MatchBranchSimplification {
         }
 
         if should_cleanup {
-            simplify_cfg(tcx, body);
+            simplify_cfg(body);
         }
     }
 }
diff --git a/compiler/rustc_mir_transform/src/promote_consts.rs b/compiler/rustc_mir_transform/src/promote_consts.rs
new file mode 100644
index 00000000000..841b86fed45
--- /dev/null
+++ b/compiler/rustc_mir_transform/src/promote_consts.rs
@@ -0,0 +1,952 @@
+//! A pass that promotes borrows of constant rvalues.
+//!
+//! The rvalues considered constant are trees of temps,
+//! each with exactly one initialization, and holding
+//! a constant value with no interior mutability.
+//! They are placed into a new MIR constant body in
+//! `promoted` and the borrow rvalue is replaced with
+//! a `Literal::Promoted` using the index into `promoted`
+//! of that constant MIR.
+//!
+//! This pass assumes that every use is dominated by an
+//! initialization and can otherwise silence errors, if
+//! move analysis runs after promotion on broken MIR.
+
+use either::{Left, Right};
+use rustc_hir as hir;
+use rustc_middle::mir;
+use rustc_middle::mir::visit::{MutVisitor, MutatingUseContext, PlaceContext, Visitor};
+use rustc_middle::mir::*;
+use rustc_middle::ty::GenericArgs;
+use rustc_middle::ty::{self, List, Ty, TyCtxt, TypeVisitableExt};
+use rustc_span::Span;
+
+use rustc_index::{Idx, IndexSlice, IndexVec};
+
+use std::assert_matches::assert_matches;
+use std::cell::Cell;
+use std::{cmp, iter, mem};
+
+use rustc_const_eval::transform::check_consts::{qualifs, ConstCx};
+
+/// A `MirPass` for promotion.
+///
+/// Promotion is the extraction of promotable temps into separate MIR bodies so they can have
+/// `'static` lifetime.
+///
+/// After this pass is run, `promoted_fragments` will hold the MIR body corresponding to each
+/// newly created `Constant`.
+#[derive(Default)]
+pub struct PromoteTemps<'tcx> {
+    pub promoted_fragments: Cell<IndexVec<Promoted, Body<'tcx>>>,
+}
+
+impl<'tcx> MirPass<'tcx> for PromoteTemps<'tcx> {
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+        // There's not really any point in promoting errorful MIR.
+        //
+        // This does not include MIR that failed const-checking, which we still try to promote.
+        if let Err(_) = body.return_ty().error_reported() {
+            debug!("PromoteTemps: MIR had errors");
+            return;
+        }
+        if body.source.promoted.is_some() {
+            return;
+        }
+
+        let ccx = ConstCx::new(tcx, body);
+        let (mut temps, all_candidates) = collect_temps_and_candidates(&ccx);
+
+        let promotable_candidates = validate_candidates(&ccx, &mut temps, &all_candidates);
+
+        let promoted = promote_candidates(body, tcx, temps, promotable_candidates);
+        self.promoted_fragments.set(promoted);
+    }
+}
+
+/// State of a temporary during collection and promotion.
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+enum TempState {
+    /// No references to this temp.
+    Undefined,
+    /// One direct assignment and any number of direct uses.
+    /// A borrow of this temp is promotable if the assigned
+    /// value is qualified as constant.
+    Defined { location: Location, uses: usize, valid: Result<(), ()> },
+    /// Any other combination of assignments/uses.
+    Unpromotable,
+    /// This temp was part of an rvalue which got extracted
+    /// during promotion and needs cleanup.
+    PromotedOut,
+}
+
+/// A "root candidate" for promotion, which will become the
+/// returned value in a promoted MIR, unless it's a subset
+/// of a larger candidate.
+#[derive(Copy, Clone, PartialEq, Eq, Debug)]
+struct Candidate {
+    location: Location,
+}
+
+struct Collector<'a, 'tcx> {
+    ccx: &'a ConstCx<'a, 'tcx>,
+    temps: IndexVec<Local, TempState>,
+    candidates: Vec<Candidate>,
+}
+
+impl<'tcx> Visitor<'tcx> for Collector<'_, 'tcx> {
+    fn visit_local(&mut self, index: Local, context: PlaceContext, location: Location) {
+        debug!("visit_local: index={:?} context={:?} location={:?}", index, context, location);
+        // We're only interested in temporaries and the return place
+        match self.ccx.body.local_kind(index) {
+            LocalKind::Arg => return,
+            LocalKind::Temp if self.ccx.body.local_decls[index].is_user_variable() => return,
+            LocalKind::ReturnPointer | LocalKind::Temp => {}
+        }
+
+        // Ignore drops, if the temp gets promoted,
+        // then it's constant and thus drop is noop.
+        // Non-uses are also irrelevant.
+        if context.is_drop() || !context.is_use() {
+            debug!(
+                "visit_local: context.is_drop={:?} context.is_use={:?}",
+                context.is_drop(),
+                context.is_use(),
+            );
+            return;
+        }
+
+        let temp = &mut self.temps[index];
+        debug!("visit_local: temp={:?}", temp);
+        *temp = match *temp {
+            TempState::Undefined => match context {
+                PlaceContext::MutatingUse(MutatingUseContext::Store)
+                | PlaceContext::MutatingUse(MutatingUseContext::Call) => {
+                    TempState::Defined { location, uses: 0, valid: Err(()) }
+                }
+                _ => TempState::Unpromotable,
+            },
+            TempState::Defined { ref mut uses, .. } => {
+                // We always allow borrows, even mutable ones, as we need
+                // to promote mutable borrows of some ZSTs e.g., `&mut []`.
+                let allowed_use = match context {
+                    PlaceContext::MutatingUse(MutatingUseContext::Borrow)
+                    | PlaceContext::NonMutatingUse(_) => true,
+                    PlaceContext::MutatingUse(_) | PlaceContext::NonUse(_) => false,
+                };
+                debug!("visit_local: allowed_use={:?}", allowed_use);
+                if allowed_use {
+                    *uses += 1;
+                    return;
+                }
+                TempState::Unpromotable
+            }
+            TempState::Unpromotable | TempState::PromotedOut => TempState::Unpromotable,
+        };
+    }
+
+    fn visit_rvalue(&mut self, rvalue: &Rvalue<'tcx>, location: Location) {
+        self.super_rvalue(rvalue, location);
+
+        if let Rvalue::Ref(..) = *rvalue {
+            self.candidates.push(Candidate { location });
+        }
+    }
+}
+
+fn collect_temps_and_candidates<'tcx>(
+    ccx: &ConstCx<'_, 'tcx>,
+) -> (IndexVec<Local, TempState>, Vec<Candidate>) {
+    let mut collector = Collector {
+        temps: IndexVec::from_elem(TempState::Undefined, &ccx.body.local_decls),
+        candidates: vec![],
+        ccx,
+    };
+    for (bb, data) in traversal::reverse_postorder(ccx.body) {
+        collector.visit_basic_block_data(bb, data);
+    }
+    (collector.temps, collector.candidates)
+}
+
+/// Checks whether locals that appear in a promotion context (`Candidate`) are actually promotable.
+///
+/// This wraps an `Item`, and has access to all fields of that `Item` via `Deref` coercion.
+struct Validator<'a, 'tcx> {
+    ccx: &'a ConstCx<'a, 'tcx>,
+    temps: &'a mut IndexSlice<Local, TempState>,
+}
+
+impl<'a, 'tcx> std::ops::Deref for Validator<'a, 'tcx> {
+    type Target = ConstCx<'a, 'tcx>;
+
+    fn deref(&self) -> &Self::Target {
+        self.ccx
+    }
+}
+
+struct Unpromotable;
+
+impl<'tcx> Validator<'_, 'tcx> {
+    fn validate_candidate(&mut self, candidate: Candidate) -> Result<(), Unpromotable> {
+        let Left(statement) = self.body.stmt_at(candidate.location) else { bug!() };
+        let Some((_, Rvalue::Ref(_, kind, place))) = statement.kind.as_assign() else { bug!() };
+
+        // We can only promote interior borrows of promotable temps (non-temps
+        // don't get promoted anyway).
+        self.validate_local(place.local)?;
+
+        // The reference operation itself must be promotable.
+        // (Needs to come after `validate_local` to avoid ICEs.)
+        self.validate_ref(*kind, place)?;
+
+        // We do not check all the projections (they do not get promoted anyway),
+        // but we do stay away from promoting anything involving a dereference.
+        if place.projection.contains(&ProjectionElem::Deref) {
+            return Err(Unpromotable);
+        }
+
+        Ok(())
+    }
+
+    // FIXME(eddyb) maybe cache this?
+    fn qualif_local<Q: qualifs::Qualif>(&mut self, local: Local) -> bool {
+        let TempState::Defined { location: loc, .. } = self.temps[local] else {
+            return false;
+        };
+
+        let stmt_or_term = self.body.stmt_at(loc);
+        match stmt_or_term {
+            Left(statement) => {
+                let Some((_, rhs)) = statement.kind.as_assign() else {
+                    span_bug!(statement.source_info.span, "{:?} is not an assignment", statement)
+                };
+                qualifs::in_rvalue::<Q, _>(self.ccx, &mut |l| self.qualif_local::<Q>(l), rhs)
+            }
+            Right(terminator) => {
+                assert_matches!(terminator.kind, TerminatorKind::Call { .. });
+                let return_ty = self.body.local_decls[local].ty;
+                Q::in_any_value_of_ty(self.ccx, return_ty)
+            }
+        }
+    }
+
+    fn validate_local(&mut self, local: Local) -> Result<(), Unpromotable> {
+        let TempState::Defined { location: loc, uses, valid } = self.temps[local] else {
+            return Err(Unpromotable);
+        };
+
+        // We cannot promote things that need dropping, since the promoted value would not get
+        // dropped.
+        if self.qualif_local::<qualifs::NeedsDrop>(local) {
+            return Err(Unpromotable);
+        }
+
+        if valid.is_ok() {
+            return Ok(());
+        }
+
+        let ok = {
+            let stmt_or_term = self.body.stmt_at(loc);
+            match stmt_or_term {
+                Left(statement) => {
+                    let Some((_, rhs)) = statement.kind.as_assign() else {
+                        span_bug!(
+                            statement.source_info.span,
+                            "{:?} is not an assignment",
+                            statement
+                        )
+                    };
+                    self.validate_rvalue(rhs)
+                }
+                Right(terminator) => match &terminator.kind {
+                    TerminatorKind::Call { func, args, .. } => self.validate_call(func, args),
+                    TerminatorKind::Yield { .. } => Err(Unpromotable),
+                    kind => {
+                        span_bug!(terminator.source_info.span, "{:?} not promotable", kind);
+                    }
+                },
+            }
+        };
+
+        self.temps[local] = match ok {
+            Ok(()) => TempState::Defined { location: loc, uses, valid: Ok(()) },
+            Err(_) => TempState::Unpromotable,
+        };
+
+        ok
+    }
+
+    fn validate_place(&mut self, place: PlaceRef<'tcx>) -> Result<(), Unpromotable> {
+        let Some((place_base, elem)) = place.last_projection() else {
+            return self.validate_local(place.local);
+        };
+
+        // Validate topmost projection, then recurse.
+        match elem {
+            // Recurse directly.
+            ProjectionElem::ConstantIndex { .. }
+            | ProjectionElem::Subtype(_)
+            | ProjectionElem::Subslice { .. } => {}
+
+            // Never recurse.
+            ProjectionElem::OpaqueCast(..) | ProjectionElem::Downcast(..) => {
+                return Err(Unpromotable);
+            }
+
+            ProjectionElem::Deref => {
+                // When a static is used by-value, that gets desugared to `*STATIC_ADDR`,
+                // and we need to be able to promote this. So check if this deref matches
+                // that specific pattern.
+
+                // We need to make sure this is a `Deref` of a local with no further projections.
+                // Discussion can be found at
+                // https://github.com/rust-lang/rust/pull/74945#discussion_r463063247
+                if let Some(local) = place_base.as_local()
+                    && let TempState::Defined { location, .. } = self.temps[local]
+                    && let Left(def_stmt) = self.body.stmt_at(location)
+                    && let Some((_, Rvalue::Use(Operand::Constant(c)))) = def_stmt.kind.as_assign()
+                    && let Some(did) = c.check_static_ptr(self.tcx)
+                    // Evaluating a promoted may not read statics except if it got
+                    // promoted from a static (this is a CTFE check). So we
+                    // can only promote static accesses inside statics.
+                    && let Some(hir::ConstContext::Static(..)) = self.const_kind
+                    && !self.tcx.is_thread_local_static(did)
+                {
+                    // Recurse.
+                } else {
+                    return Err(Unpromotable);
+                }
+            }
+            ProjectionElem::Index(local) => {
+                // Only accept if we can predict the index and are indexing an array.
+                if let TempState::Defined { location: loc, .. } = self.temps[local]
+                    && let Left(statement) =  self.body.stmt_at(loc)
+                    && let Some((_, Rvalue::Use(Operand::Constant(c)))) = statement.kind.as_assign()
+                    && let Some(idx) = c.const_.try_eval_target_usize(self.tcx, self.param_env)
+                    // Determine the type of the thing we are indexing.
+                    && let ty::Array(_, len) = place_base.ty(self.body, self.tcx).ty.kind()
+                    // It's an array; determine its length.
+                    && let Some(len) = len.try_eval_target_usize(self.tcx, self.param_env)
+                    // If the index is in-bounds, go ahead.
+                    && idx < len
+                {
+                    self.validate_local(local)?;
+                    // Recurse.
+                } else {
+                    return Err(Unpromotable);
+                }
+            }
+
+            ProjectionElem::Field(..) => {
+                let base_ty = place_base.ty(self.body, self.tcx).ty;
+                if base_ty.is_union() {
+                    // No promotion of union field accesses.
+                    return Err(Unpromotable);
+                }
+            }
+        }
+
+        self.validate_place(place_base)
+    }
+
+    fn validate_operand(&mut self, operand: &Operand<'tcx>) -> Result<(), Unpromotable> {
+        match operand {
+            Operand::Copy(place) | Operand::Move(place) => self.validate_place(place.as_ref()),
+
+            // The qualifs for a constant (e.g. `HasMutInterior`) are checked in
+            // `validate_rvalue` upon access.
+            Operand::Constant(c) => {
+                if let Some(def_id) = c.check_static_ptr(self.tcx) {
+                    // Only allow statics (not consts) to refer to other statics.
+                    // FIXME(eddyb) does this matter at all for promotion?
+                    // FIXME(RalfJung) it makes little sense to not promote this in `fn`/`const fn`,
+                    // and in `const` this cannot occur anyway. The only concern is that we might
+                    // promote even `let x = &STATIC` which would be useless, but this applies to
+                    // promotion inside statics as well.
+                    let is_static = matches!(self.const_kind, Some(hir::ConstContext::Static(_)));
+                    if !is_static {
+                        return Err(Unpromotable);
+                    }
+
+                    let is_thread_local = self.tcx.is_thread_local_static(def_id);
+                    if is_thread_local {
+                        return Err(Unpromotable);
+                    }
+                }
+
+                Ok(())
+            }
+        }
+    }
+
+    fn validate_ref(&mut self, kind: BorrowKind, place: &Place<'tcx>) -> Result<(), Unpromotable> {
+        match kind {
+            // Reject these borrow types just to be safe.
+            // FIXME(RalfJung): could we allow them? Should we? No point in it until we have a usecase.
+            BorrowKind::Fake | BorrowKind::Mut { kind: MutBorrowKind::ClosureCapture } => {
+                return Err(Unpromotable);
+            }
+
+            BorrowKind::Shared => {
+                let has_mut_interior = self.qualif_local::<qualifs::HasMutInterior>(place.local);
+                if has_mut_interior {
+                    return Err(Unpromotable);
+                }
+            }
+
+            // FIXME: consider changing this to only promote &mut [] for default borrows,
+            // also forbidding two phase borrows
+            BorrowKind::Mut { kind: MutBorrowKind::Default | MutBorrowKind::TwoPhaseBorrow } => {
+                let ty = place.ty(self.body, self.tcx).ty;
+
+                // In theory, any zero-sized value could be borrowed
+                // mutably without consequences. However, only &mut []
+                // is allowed right now.
+                if let ty::Array(_, len) = ty.kind() {
+                    match len.try_eval_target_usize(self.tcx, self.param_env) {
+                        Some(0) => {}
+                        _ => return Err(Unpromotable),
+                    }
+                } else {
+                    return Err(Unpromotable);
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    fn validate_rvalue(&mut self, rvalue: &Rvalue<'tcx>) -> Result<(), Unpromotable> {
+        match rvalue {
+            Rvalue::Use(operand) | Rvalue::Repeat(operand, _) => {
+                self.validate_operand(operand)?;
+            }
+            Rvalue::CopyForDeref(place) => {
+                let op = &Operand::Copy(*place);
+                self.validate_operand(op)?
+            }
+
+            Rvalue::Discriminant(place) | Rvalue::Len(place) => {
+                self.validate_place(place.as_ref())?
+            }
+
+            Rvalue::ThreadLocalRef(_) => return Err(Unpromotable),
+
+            // ptr-to-int casts are not possible in consts and thus not promotable
+            Rvalue::Cast(CastKind::PointerExposeAddress, _, _) => return Err(Unpromotable),
+
+            // all other casts including int-to-ptr casts are fine, they just use the integer value
+            // at pointer type.
+            Rvalue::Cast(_, operand, _) => {
+                self.validate_operand(operand)?;
+            }
+
+            Rvalue::NullaryOp(op, _) => match op {
+                NullOp::SizeOf => {}
+                NullOp::AlignOf => {}
+                NullOp::OffsetOf(_) => {}
+            },
+
+            Rvalue::ShallowInitBox(_, _) => return Err(Unpromotable),
+
+            Rvalue::UnaryOp(op, operand) => {
+                match op {
+                    // These operations can never fail.
+                    UnOp::Neg | UnOp::Not => {}
+                }
+
+                self.validate_operand(operand)?;
+            }
+
+            Rvalue::BinaryOp(op, box (lhs, rhs)) | Rvalue::CheckedBinaryOp(op, box (lhs, rhs)) => {
+                let op = *op;
+                let lhs_ty = lhs.ty(self.body, self.tcx);
+
+                if let ty::RawPtr(_) | ty::FnPtr(..) = lhs_ty.kind() {
+                    // Raw and fn pointer operations are not allowed inside consts and thus not promotable.
+                    assert!(matches!(
+                        op,
+                        BinOp::Eq
+                            | BinOp::Ne
+                            | BinOp::Le
+                            | BinOp::Lt
+                            | BinOp::Ge
+                            | BinOp::Gt
+                            | BinOp::Offset
+                    ));
+                    return Err(Unpromotable);
+                }
+
+                match op {
+                    BinOp::Div | BinOp::Rem => {
+                        if lhs_ty.is_integral() {
+                            // Integer division: the RHS must be a non-zero const.
+                            let const_val = match rhs {
+                                Operand::Constant(c) => {
+                                    c.const_.try_eval_bits(self.tcx, self.param_env)
+                                }
+                                _ => None,
+                            };
+                            match const_val {
+                                Some(x) if x != 0 => {}        // okay
+                                _ => return Err(Unpromotable), // value not known or 0 -- not okay
+                            }
+                        }
+                    }
+                    // The remaining operations can never fail.
+                    BinOp::Eq
+                    | BinOp::Ne
+                    | BinOp::Le
+                    | BinOp::Lt
+                    | BinOp::Ge
+                    | BinOp::Gt
+                    | BinOp::Offset
+                    | BinOp::Add
+                    | BinOp::AddUnchecked
+                    | BinOp::Sub
+                    | BinOp::SubUnchecked
+                    | BinOp::Mul
+                    | BinOp::MulUnchecked
+                    | BinOp::BitXor
+                    | BinOp::BitAnd
+                    | BinOp::BitOr
+                    | BinOp::Shl
+                    | BinOp::ShlUnchecked
+                    | BinOp::Shr
+                    | BinOp::ShrUnchecked => {}
+                }
+
+                self.validate_operand(lhs)?;
+                self.validate_operand(rhs)?;
+            }
+
+            Rvalue::AddressOf(_, place) => {
+                // We accept `&raw *`, i.e., raw reborrows -- creating a raw pointer is
+                // no problem, only using it is.
+                if let Some((place_base, ProjectionElem::Deref)) = place.as_ref().last_projection()
+                {
+                    let base_ty = place_base.ty(self.body, self.tcx).ty;
+                    if let ty::Ref(..) = base_ty.kind() {
+                        return self.validate_place(place_base);
+                    }
+                }
+                return Err(Unpromotable);
+            }
+
+            Rvalue::Ref(_, kind, place) => {
+                // Special-case reborrows to be more like a copy of the reference.
+                let mut place_simplified = place.as_ref();
+                if let Some((place_base, ProjectionElem::Deref)) =
+                    place_simplified.last_projection()
+                {
+                    let base_ty = place_base.ty(self.body, self.tcx).ty;
+                    if let ty::Ref(..) = base_ty.kind() {
+                        place_simplified = place_base;
+                    }
+                }
+
+                self.validate_place(place_simplified)?;
+
+                // Check that the reference is fine (using the original place!).
+                // (Needs to come after `validate_place` to avoid ICEs.)
+                self.validate_ref(*kind, place)?;
+            }
+
+            Rvalue::Aggregate(_, operands) => {
+                for o in operands {
+                    self.validate_operand(o)?;
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    fn validate_call(
+        &mut self,
+        callee: &Operand<'tcx>,
+        args: &[Operand<'tcx>],
+    ) -> Result<(), Unpromotable> {
+        let fn_ty = callee.ty(self.body, self.tcx);
+
+        // Inside const/static items, we promote all (eligible) function calls.
+        // Everywhere else, we require `#[rustc_promotable]` on the callee.
+        let promote_all_const_fn = matches!(
+            self.const_kind,
+            Some(hir::ConstContext::Static(_) | hir::ConstContext::Const { inline: false })
+        );
+        if !promote_all_const_fn {
+            if let ty::FnDef(def_id, _) = *fn_ty.kind() {
+                // Never promote runtime `const fn` calls of
+                // functions without `#[rustc_promotable]`.
+                if !self.tcx.is_promotable_const_fn(def_id) {
+                    return Err(Unpromotable);
+                }
+            }
+        }
+
+        let is_const_fn = match *fn_ty.kind() {
+            ty::FnDef(def_id, _) => self.tcx.is_const_fn_raw(def_id),
+            _ => false,
+        };
+        if !is_const_fn {
+            return Err(Unpromotable);
+        }
+
+        self.validate_operand(callee)?;
+        for arg in args {
+            self.validate_operand(arg)?;
+        }
+
+        Ok(())
+    }
+}
+
+// FIXME(eddyb) remove the differences for promotability in `static`, `const`, `const fn`.
+fn validate_candidates(
+    ccx: &ConstCx<'_, '_>,
+    temps: &mut IndexSlice<Local, TempState>,
+    candidates: &[Candidate],
+) -> Vec<Candidate> {
+    let mut validator = Validator { ccx, temps };
+
+    candidates
+        .iter()
+        .copied()
+        .filter(|&candidate| validator.validate_candidate(candidate).is_ok())
+        .collect()
+}
+
+struct Promoter<'a, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    source: &'a mut Body<'tcx>,
+    promoted: Body<'tcx>,
+    temps: &'a mut IndexVec<Local, TempState>,
+    extra_statements: &'a mut Vec<(Location, Statement<'tcx>)>,
+
+    /// If true, all nested temps are also kept in the
+    /// source MIR, not moved to the promoted MIR.
+    keep_original: bool,
+}
+
+impl<'a, 'tcx> Promoter<'a, 'tcx> {
+    fn new_block(&mut self) -> BasicBlock {
+        let span = self.promoted.span;
+        self.promoted.basic_blocks_mut().push(BasicBlockData {
+            statements: vec![],
+            terminator: Some(Terminator {
+                source_info: SourceInfo::outermost(span),
+                kind: TerminatorKind::Return,
+            }),
+            is_cleanup: false,
+        })
+    }
+
+    fn assign(&mut self, dest: Local, rvalue: Rvalue<'tcx>, span: Span) {
+        let last = self.promoted.basic_blocks.last_index().unwrap();
+        let data = &mut self.promoted[last];
+        data.statements.push(Statement {
+            source_info: SourceInfo::outermost(span),
+            kind: StatementKind::Assign(Box::new((Place::from(dest), rvalue))),
+        });
+    }
+
+    fn is_temp_kind(&self, local: Local) -> bool {
+        self.source.local_kind(local) == LocalKind::Temp
+    }
+
+    /// Copies the initialization of this temp to the
+    /// promoted MIR, recursing through temps.
+    fn promote_temp(&mut self, temp: Local) -> Local {
+        let old_keep_original = self.keep_original;
+        let loc = match self.temps[temp] {
+            TempState::Defined { location, uses, .. } if uses > 0 => {
+                if uses > 1 {
+                    self.keep_original = true;
+                }
+                location
+            }
+            state => {
+                span_bug!(self.promoted.span, "{:?} not promotable: {:?}", temp, state);
+            }
+        };
+        if !self.keep_original {
+            self.temps[temp] = TempState::PromotedOut;
+        }
+
+        let num_stmts = self.source[loc.block].statements.len();
+        let new_temp = self.promoted.local_decls.push(LocalDecl::new(
+            self.source.local_decls[temp].ty,
+            self.source.local_decls[temp].source_info.span,
+        ));
+
+        debug!("promote({:?} @ {:?}/{:?}, {:?})", temp, loc, num_stmts, self.keep_original);
+
+        // First, take the Rvalue or Call out of the source MIR,
+        // or duplicate it, depending on keep_original.
+        if loc.statement_index < num_stmts {
+            let (mut rvalue, source_info) = {
+                let statement = &mut self.source[loc.block].statements[loc.statement_index];
+                let StatementKind::Assign(box (_, rhs)) = &mut statement.kind else {
+                    span_bug!(statement.source_info.span, "{:?} is not an assignment", statement);
+                };
+
+                (
+                    if self.keep_original {
+                        rhs.clone()
+                    } else {
+                        let unit = Rvalue::Use(Operand::Constant(Box::new(ConstOperand {
+                            span: statement.source_info.span,
+                            user_ty: None,
+                            const_: Const::zero_sized(self.tcx.types.unit),
+                        })));
+                        mem::replace(rhs, unit)
+                    },
+                    statement.source_info,
+                )
+            };
+
+            self.visit_rvalue(&mut rvalue, loc);
+            self.assign(new_temp, rvalue, source_info.span);
+        } else {
+            let terminator = if self.keep_original {
+                self.source[loc.block].terminator().clone()
+            } else {
+                let terminator = self.source[loc.block].terminator_mut();
+                let target = match &terminator.kind {
+                    TerminatorKind::Call { target: Some(target), .. } => *target,
+                    kind => {
+                        span_bug!(terminator.source_info.span, "{:?} not promotable", kind);
+                    }
+                };
+                Terminator {
+                    source_info: terminator.source_info,
+                    kind: mem::replace(&mut terminator.kind, TerminatorKind::Goto { target }),
+                }
+            };
+
+            match terminator.kind {
+                TerminatorKind::Call {
+                    mut func, mut args, call_source: desugar, fn_span, ..
+                } => {
+                    self.visit_operand(&mut func, loc);
+                    for arg in &mut args {
+                        self.visit_operand(arg, loc);
+                    }
+
+                    let last = self.promoted.basic_blocks.last_index().unwrap();
+                    let new_target = self.new_block();
+
+                    *self.promoted[last].terminator_mut() = Terminator {
+                        kind: TerminatorKind::Call {
+                            func,
+                            args,
+                            unwind: UnwindAction::Continue,
+                            destination: Place::from(new_temp),
+                            target: Some(new_target),
+                            call_source: desugar,
+                            fn_span,
+                        },
+                        source_info: SourceInfo::outermost(terminator.source_info.span),
+                        ..terminator
+                    };
+                }
+                kind => {
+                    span_bug!(terminator.source_info.span, "{:?} not promotable", kind);
+                }
+            };
+        };
+
+        self.keep_original = old_keep_original;
+        new_temp
+    }
+
+    fn promote_candidate(mut self, candidate: Candidate, next_promoted_id: usize) -> Body<'tcx> {
+        let def = self.source.source.def_id();
+        let mut rvalue = {
+            let promoted = &mut self.promoted;
+            let promoted_id = Promoted::new(next_promoted_id);
+            let tcx = self.tcx;
+            let mut promoted_operand = |ty, span| {
+                promoted.span = span;
+                promoted.local_decls[RETURN_PLACE] = LocalDecl::new(ty, span);
+                let args = tcx.erase_regions(GenericArgs::identity_for_item(tcx, def));
+                let uneval = mir::UnevaluatedConst { def, args, promoted: Some(promoted_id) };
+
+                Operand::Constant(Box::new(ConstOperand {
+                    span,
+                    user_ty: None,
+                    const_: Const::Unevaluated(uneval, ty),
+                }))
+            };
+
+            let blocks = self.source.basic_blocks.as_mut();
+            let local_decls = &mut self.source.local_decls;
+            let loc = candidate.location;
+            let statement = &mut blocks[loc.block].statements[loc.statement_index];
+            let StatementKind::Assign(box (_, Rvalue::Ref(region, borrow_kind, place))) =
+                &mut statement.kind
+            else {
+                bug!()
+            };
+
+            // Use the underlying local for this (necessarily interior) borrow.
+            debug_assert!(region.is_erased());
+            let ty = local_decls[place.local].ty;
+            let span = statement.source_info.span;
+
+            let ref_ty = Ty::new_ref(
+                tcx,
+                tcx.lifetimes.re_erased,
+                ty::TypeAndMut { ty, mutbl: borrow_kind.to_mutbl_lossy() },
+            );
+
+            let mut projection = vec![PlaceElem::Deref];
+            projection.extend(place.projection);
+            place.projection = tcx.mk_place_elems(&projection);
+
+            // Create a temp to hold the promoted reference.
+            // This is because `*r` requires `r` to be a local,
+            // otherwise we would use the `promoted` directly.
+            let mut promoted_ref = LocalDecl::new(ref_ty, span);
+            promoted_ref.source_info = statement.source_info;
+            let promoted_ref = local_decls.push(promoted_ref);
+            assert_eq!(self.temps.push(TempState::Unpromotable), promoted_ref);
+
+            let promoted_ref_statement = Statement {
+                source_info: statement.source_info,
+                kind: StatementKind::Assign(Box::new((
+                    Place::from(promoted_ref),
+                    Rvalue::Use(promoted_operand(ref_ty, span)),
+                ))),
+            };
+            self.extra_statements.push((loc, promoted_ref_statement));
+
+            Rvalue::Ref(
+                tcx.lifetimes.re_erased,
+                *borrow_kind,
+                Place {
+                    local: mem::replace(&mut place.local, promoted_ref),
+                    projection: List::empty(),
+                },
+            )
+        };
+
+        assert_eq!(self.new_block(), START_BLOCK);
+        self.visit_rvalue(
+            &mut rvalue,
+            Location { block: START_BLOCK, statement_index: usize::MAX },
+        );
+
+        let span = self.promoted.span;
+        self.assign(RETURN_PLACE, rvalue, span);
+        self.promoted
+    }
+}
+
+/// Replaces all temporaries with their promoted counterparts.
+impl<'a, 'tcx> MutVisitor<'tcx> for Promoter<'a, 'tcx> {
+    fn tcx(&self) -> TyCtxt<'tcx> {
+        self.tcx
+    }
+
+    fn visit_local(&mut self, local: &mut Local, _: PlaceContext, _: Location) {
+        if self.is_temp_kind(*local) {
+            *local = self.promote_temp(*local);
+        }
+    }
+}
+
+fn promote_candidates<'tcx>(
+    body: &mut Body<'tcx>,
+    tcx: TyCtxt<'tcx>,
+    mut temps: IndexVec<Local, TempState>,
+    candidates: Vec<Candidate>,
+) -> IndexVec<Promoted, Body<'tcx>> {
+    // Visit candidates in reverse, in case they're nested.
+    debug!("promote_candidates({:?})", candidates);
+
+    let mut promotions = IndexVec::new();
+
+    let mut extra_statements = vec![];
+    for candidate in candidates.into_iter().rev() {
+        let Location { block, statement_index } = candidate.location;
+        if let StatementKind::Assign(box (place, _)) = &body[block].statements[statement_index].kind
+        {
+            if let Some(local) = place.as_local() {
+                if temps[local] == TempState::PromotedOut {
+                    // Already promoted.
+                    continue;
+                }
+            }
+        }
+
+        // Declare return place local so that `mir::Body::new` doesn't complain.
+        let initial_locals = iter::once(LocalDecl::new(tcx.types.never, body.span)).collect();
+
+        let mut scope = body.source_scopes[body.source_info(candidate.location).scope].clone();
+        scope.parent_scope = None;
+
+        let mut promoted = Body::new(
+            body.source, // `promoted` gets filled in below
+            IndexVec::new(),
+            IndexVec::from_elem_n(scope, 1),
+            initial_locals,
+            IndexVec::new(),
+            0,
+            vec![],
+            body.span,
+            None,
+            body.tainted_by_errors,
+        );
+        promoted.phase = MirPhase::Analysis(AnalysisPhase::Initial);
+
+        let promoter = Promoter {
+            promoted,
+            tcx,
+            source: body,
+            temps: &mut temps,
+            extra_statements: &mut extra_statements,
+            keep_original: false,
+        };
+
+        let mut promoted = promoter.promote_candidate(candidate, promotions.len());
+        promoted.source.promoted = Some(promotions.next_index());
+        promotions.push(promoted);
+    }
+
+    // Insert each of `extra_statements` before its indicated location, which
+    // has to be done in reverse location order, to not invalidate the rest.
+    extra_statements.sort_by_key(|&(loc, _)| cmp::Reverse(loc));
+    for (loc, statement) in extra_statements {
+        body[loc.block].statements.insert(loc.statement_index, statement);
+    }
+
+    // Eliminate assignments to, and drops of promoted temps.
+    let promoted = |index: Local| temps[index] == TempState::PromotedOut;
+    for block in body.basic_blocks_mut() {
+        block.statements.retain(|statement| match &statement.kind {
+            StatementKind::Assign(box (place, _)) => {
+                if let Some(index) = place.as_local() {
+                    !promoted(index)
+                } else {
+                    true
+                }
+            }
+            StatementKind::StorageLive(index) | StatementKind::StorageDead(index) => {
+                !promoted(*index)
+            }
+            _ => true,
+        });
+        let terminator = block.terminator_mut();
+        if let TerminatorKind::Drop { place, target, .. } = &terminator.kind {
+            if let Some(index) = place.as_local() {
+                if promoted(index) {
+                    terminator.kind = TerminatorKind::Goto { target: *target };
+                }
+            }
+        }
+    }
+
+    promotions
+}
diff --git a/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs b/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs
index 5d528bed356..2778d91e17b 100644
--- a/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs
+++ b/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs
@@ -38,7 +38,7 @@ impl<'tcx> MirPass<'tcx> for RemoveUnneededDrops {
         // if we applied optimizations, we potentially have some cfg to cleanup to
         // make it easier for further passes
         if should_simplify {
-            simplify_cfg(tcx, body);
+            simplify_cfg(body);
         }
     }
 }
diff --git a/compiler/rustc_mir_transform/src/separate_const_switch.rs b/compiler/rustc_mir_transform/src/separate_const_switch.rs
index 6e22690d8da..7120ef72142 100644
--- a/compiler/rustc_mir_transform/src/separate_const_switch.rs
+++ b/compiler/rustc_mir_transform/src/separate_const_switch.rs
@@ -50,11 +50,11 @@ impl<'tcx> MirPass<'tcx> for SeparateConstSwitch {
         sess.mir_opt_level() >= 2 && sess.opts.unstable_opts.unsound_mir_opts
     }
 
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, _: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         // If execution did something, applying a simplification layer
         // helps later passes optimize the copy away.
         if separate_const_switch(body) > 0 {
-            super::simplify::simplify_cfg(tcx, body);
+            super::simplify::simplify_cfg(body);
         }
     }
 }
diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs
index 856a0f22771..8c8818bd68e 100644
--- a/compiler/rustc_mir_transform/src/simplify.rs
+++ b/compiler/rustc_mir_transform/src/simplify.rs
@@ -27,7 +27,6 @@
 //! naively generate still contains the `_a = ()` write in the unreachable block "after" the
 //! return.
 
-use rustc_data_structures::fx::FxIndexSet;
 use rustc_index::{Idx, IndexSlice, IndexVec};
 use rustc_middle::mir::visit::{MutVisitor, MutatingUseContext, PlaceContext, Visitor};
 use rustc_middle::mir::*;
@@ -62,9 +61,8 @@ impl SimplifyCfg {
     }
 }
 
-pub fn simplify_cfg<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+pub(crate) fn simplify_cfg(body: &mut Body<'_>) {
     CfgSimplifier::new(body).simplify();
-    remove_duplicate_unreachable_blocks(tcx, body);
     remove_dead_blocks(body);
 
     // FIXME: Should probably be moved into some kind of pass manager
@@ -76,9 +74,9 @@ impl<'tcx> MirPass<'tcx> for SimplifyCfg {
         self.name()
     }
 
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    fn run_pass(&self, _: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         debug!("SimplifyCfg({:?}) - simplifying {:?}", self.name(), body.source);
-        simplify_cfg(tcx, body);
+        simplify_cfg(body);
     }
 }
 
@@ -289,55 +287,25 @@ pub fn simplify_duplicate_switch_targets(terminator: &mut Terminator<'_>) {
     }
 }
 
-pub fn remove_duplicate_unreachable_blocks<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-    struct OptApplier<'tcx> {
-        tcx: TyCtxt<'tcx>,
-        duplicates: FxIndexSet<BasicBlock>,
-    }
-
-    impl<'tcx> MutVisitor<'tcx> for OptApplier<'tcx> {
-        fn tcx(&self) -> TyCtxt<'tcx> {
-            self.tcx
-        }
-
-        fn visit_terminator(&mut self, terminator: &mut Terminator<'tcx>, location: Location) {
-            for target in terminator.successors_mut() {
-                // We don't have to check whether `target` is a cleanup block, because have
-                // entirely excluded cleanup blocks in building the set of duplicates.
-                if self.duplicates.contains(target) {
-                    *target = self.duplicates[0];
-                }
-            }
-
-            simplify_duplicate_switch_targets(terminator);
-
-            self.super_terminator(terminator, location);
-        }
-    }
+pub(crate) fn remove_dead_blocks(body: &mut Body<'_>) {
+    let should_deduplicate_unreachable = |bbdata: &BasicBlockData<'_>| {
+        // CfgSimplifier::simplify leaves behind some unreachable basic blocks without a
+        // terminator. Those blocks will be deleted by remove_dead_blocks, but we run just
+        // before then so we need to handle missing terminators.
+        // We also need to prevent confusing cleanup and non-cleanup blocks. In practice we
+        // don't emit empty unreachable cleanup blocks, so this simple check suffices.
+        bbdata.terminator.is_some() && bbdata.is_empty_unreachable() && !bbdata.is_cleanup
+    };
 
-    let unreachable_blocks = body
+    let reachable = traversal::reachable_as_bitset(body);
+    let empty_unreachable_blocks = body
         .basic_blocks
         .iter_enumerated()
-        .filter(|(_, bb)| {
-            // CfgSimplifier::simplify leaves behind some unreachable basic blocks without a
-            // terminator. Those blocks will be deleted by remove_dead_blocks, but we run just
-            // before then so we need to handle missing terminators.
-            // We also need to prevent confusing cleanup and non-cleanup blocks. In practice we
-            // don't emit empty unreachable cleanup blocks, so this simple check suffices.
-            bb.terminator.is_some() && bb.is_empty_unreachable() && !bb.is_cleanup
-        })
-        .map(|(block, _)| block)
-        .collect::<FxIndexSet<_>>();
-
-    if unreachable_blocks.len() > 1 {
-        OptApplier { tcx, duplicates: unreachable_blocks }.visit_body(body);
-    }
-}
+        .filter(|(bb, bbdata)| should_deduplicate_unreachable(bbdata) && reachable.contains(*bb))
+        .count();
 
-pub fn remove_dead_blocks(body: &mut Body<'_>) {
-    let reachable = traversal::reachable_as_bitset(body);
     let num_blocks = body.basic_blocks.len();
-    if num_blocks == reachable.count() {
+    if num_blocks == reachable.count() && empty_unreachable_blocks <= 1 {
         return;
     }
 
@@ -346,14 +314,28 @@ pub fn remove_dead_blocks(body: &mut Body<'_>) {
     let mut replacements: Vec<_> = (0..num_blocks).map(BasicBlock::new).collect();
     let mut orig_index = 0;
     let mut used_index = 0;
-    basic_blocks.raw.retain(|_| {
-        let keep = reachable.contains(BasicBlock::new(orig_index));
-        if keep {
-            replacements[orig_index] = BasicBlock::new(used_index);
-            used_index += 1;
+    let mut kept_unreachable = None;
+    basic_blocks.raw.retain(|bbdata| {
+        let orig_bb = BasicBlock::new(orig_index);
+        if !reachable.contains(orig_bb) {
+            orig_index += 1;
+            return false;
+        }
+
+        let used_bb = BasicBlock::new(used_index);
+        if should_deduplicate_unreachable(bbdata) {
+            let kept_unreachable = *kept_unreachable.get_or_insert(used_bb);
+            if kept_unreachable != used_bb {
+                replacements[orig_index] = kept_unreachable;
+                orig_index += 1;
+                return false;
+            }
         }
+
+        replacements[orig_index] = used_bb;
+        used_index += 1;
         orig_index += 1;
-        keep
+        true
     });
 
     for block in basic_blocks {