Merge from rustc

34 files changed, 1339 insertions, 903 deletions

diff --git a/compiler/rustc_mir_transform/src/copy_prop.rs b/compiler/rustc_mir_transform/src/copy_prop.rs
index 9b3443d3209..f149bf97cde 100644
--- a/compiler/rustc_mir_transform/src/copy_prop.rs
+++ b/compiler/rustc_mir_transform/src/copy_prop.rs
@@ -1,5 +1,5 @@
 use rustc_index::IndexSlice;
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
@@ -34,7 +34,7 @@ impl<'tcx> crate::MirPass<'tcx> for CopyProp {
         let fully_moved = fully_moved_locals(&ssa, body);
         debug!(?fully_moved);
 
-        let mut storage_to_remove = BitSet::new_empty(fully_moved.domain_size());
+        let mut storage_to_remove = DenseBitSet::new_empty(fully_moved.domain_size());
         for (local, &head) in ssa.copy_classes().iter_enumerated() {
             if local != head {
                 storage_to_remove.insert(head);
@@ -68,8 +68,8 @@ impl<'tcx> crate::MirPass<'tcx> for CopyProp {
 /// This means that replacing it by a copy of `_a` if ok, since this copy happens before `_c` is
 /// moved, and therefore that `_d` is moved.
 #[instrument(level = "trace", skip(ssa, body))]
-fn fully_moved_locals(ssa: &SsaLocals, body: &Body<'_>) -> BitSet<Local> {
-    let mut fully_moved = BitSet::new_filled(body.local_decls.len());
+fn fully_moved_locals(ssa: &SsaLocals, body: &Body<'_>) -> DenseBitSet<Local> {
+    let mut fully_moved = DenseBitSet::new_filled(body.local_decls.len());
 
     for (_, rvalue, _) in ssa.assignments(body) {
         let (Rvalue::Use(Operand::Copy(place) | Operand::Move(place))
@@ -96,9 +96,9 @@ fn fully_moved_locals(ssa: &SsaLocals, body: &Body<'_>) -> BitSet<Local> {
 /// Utility to help performing substitution of `*pattern` by `target`.
 struct Replacer<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
-    fully_moved: BitSet<Local>,
-    storage_to_remove: BitSet<Local>,
-    borrowed_locals: &'a BitSet<Local>,
+    fully_moved: DenseBitSet<Local>,
+    storage_to_remove: DenseBitSet<Local>,
+    borrowed_locals: &'a DenseBitSet<Local>,
     copy_classes: &'a IndexSlice<Local, Local>,
 }
 
diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs
index f6536d78761..a3715b5d485 100644
--- a/compiler/rustc_mir_transform/src/coroutine.rs
+++ b/compiler/rustc_mir_transform/src/coroutine.rs
@@ -60,7 +60,7 @@ use rustc_errors::pluralize;
 use rustc_hir as hir;
 use rustc_hir::lang_items::LangItem;
 use rustc_hir::{CoroutineDesugaring, CoroutineKind};
-use rustc_index::bit_set::{BitMatrix, BitSet, GrowableBitSet};
+use rustc_index::bit_set::{BitMatrix, DenseBitSet, GrowableBitSet};
 use rustc_index::{Idx, IndexVec};
 use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor};
 use rustc_middle::mir::*;
@@ -185,13 +185,13 @@ struct TransformVisitor<'tcx> {
     remap: IndexVec<Local, Option<(Ty<'tcx>, VariantIdx, FieldIdx)>>,
 
     // A map from a suspension point in a block to the locals which have live storage at that point
-    storage_liveness: IndexVec<BasicBlock, Option<BitSet<Local>>>,
+    storage_liveness: IndexVec<BasicBlock, Option<DenseBitSet<Local>>>,
 
     // A list of suspension points, generated during the transform
     suspension_points: Vec<SuspensionPoint<'tcx>>,
 
     // The set of locals that have no `StorageLive`/`StorageDead` annotations.
-    always_live_locals: BitSet<Local>,
+    always_live_locals: DenseBitSet<Local>,
 
     // The original RETURN_PLACE local
     old_ret_local: Local,
@@ -633,7 +633,7 @@ struct LivenessInfo {
     saved_locals: CoroutineSavedLocals,
 
     /// The set of saved locals live at each suspension point.
-    live_locals_at_suspension_points: Vec<BitSet<CoroutineSavedLocal>>,
+    live_locals_at_suspension_points: Vec<DenseBitSet<CoroutineSavedLocal>>,
 
     /// Parallel vec to the above with SourceInfo for each yield terminator.
     source_info_at_suspension_points: Vec<SourceInfo>,
@@ -645,7 +645,7 @@ struct LivenessInfo {
 
     /// For every suspending block, the locals which are storage-live across
     /// that suspension point.
-    storage_liveness: IndexVec<BasicBlock, Option<BitSet<Local>>>,
+    storage_liveness: IndexVec<BasicBlock, Option<DenseBitSet<Local>>>,
 }
 
 /// Computes which locals have to be stored in the state-machine for the
@@ -659,7 +659,7 @@ struct LivenessInfo {
 fn locals_live_across_suspend_points<'tcx>(
     tcx: TyCtxt<'tcx>,
     body: &Body<'tcx>,
-    always_live_locals: &BitSet<Local>,
+    always_live_locals: &DenseBitSet<Local>,
     movable: bool,
 ) -> LivenessInfo {
     // Calculate when MIR locals have live storage. This gives us an upper bound of their
@@ -688,7 +688,7 @@ fn locals_live_across_suspend_points<'tcx>(
     let mut storage_liveness_map = IndexVec::from_elem(None, &body.basic_blocks);
     let mut live_locals_at_suspension_points = Vec::new();
     let mut source_info_at_suspension_points = Vec::new();
-    let mut live_locals_at_any_suspension_point = BitSet::new_empty(body.local_decls.len());
+    let mut live_locals_at_any_suspension_point = DenseBitSet::new_empty(body.local_decls.len());
 
     for (block, data) in body.basic_blocks.iter_enumerated() {
         if let TerminatorKind::Yield { .. } = data.terminator().kind {
@@ -768,7 +768,7 @@ fn locals_live_across_suspend_points<'tcx>(
 /// `CoroutineSavedLocal` is indexed in terms of the elements in this set;
 /// i.e. `CoroutineSavedLocal::new(1)` corresponds to the second local
 /// included in this set.
-struct CoroutineSavedLocals(BitSet<Local>);
+struct CoroutineSavedLocals(DenseBitSet<Local>);
 
 impl CoroutineSavedLocals {
     /// Returns an iterator over each `CoroutineSavedLocal` along with the `Local` it corresponds
@@ -777,11 +777,11 @@ impl CoroutineSavedLocals {
         self.iter().enumerate().map(|(i, l)| (CoroutineSavedLocal::from(i), l))
     }
 
-    /// Transforms a `BitSet<Local>` that contains only locals saved across yield points to the
-    /// equivalent `BitSet<CoroutineSavedLocal>`.
-    fn renumber_bitset(&self, input: &BitSet<Local>) -> BitSet<CoroutineSavedLocal> {
+    /// Transforms a `DenseBitSet<Local>` that contains only locals saved across yield points to the
+    /// equivalent `DenseBitSet<CoroutineSavedLocal>`.
+    fn renumber_bitset(&self, input: &DenseBitSet<Local>) -> DenseBitSet<CoroutineSavedLocal> {
         assert!(self.superset(input), "{:?} not a superset of {:?}", self.0, input);
-        let mut out = BitSet::new_empty(self.count());
+        let mut out = DenseBitSet::new_empty(self.count());
         for (saved_local, local) in self.iter_enumerated() {
             if input.contains(local) {
                 out.insert(saved_local);
@@ -801,7 +801,7 @@ impl CoroutineSavedLocals {
 }
 
 impl ops::Deref for CoroutineSavedLocals {
-    type Target = BitSet<Local>;
+    type Target = DenseBitSet<Local>;
 
     fn deref(&self) -> &Self::Target {
         &self.0
@@ -815,7 +815,7 @@ impl ops::Deref for CoroutineSavedLocals {
 fn compute_storage_conflicts<'mir, 'tcx>(
     body: &'mir Body<'tcx>,
     saved_locals: &'mir CoroutineSavedLocals,
-    always_live_locals: BitSet<Local>,
+    always_live_locals: DenseBitSet<Local>,
     mut requires_storage: Results<'tcx, MaybeRequiresStorage<'mir, 'tcx>>,
 ) -> BitMatrix<CoroutineSavedLocal, CoroutineSavedLocal> {
     assert_eq!(body.local_decls.len(), saved_locals.domain_size());
@@ -833,7 +833,7 @@ fn compute_storage_conflicts<'mir, 'tcx>(
         body,
         saved_locals,
         local_conflicts: BitMatrix::from_row_n(&ineligible_locals, body.local_decls.len()),
-        eligible_storage_live: BitSet::new_empty(body.local_decls.len()),
+        eligible_storage_live: DenseBitSet::new_empty(body.local_decls.len()),
     };
 
     requires_storage.visit_reachable_with(body, &mut visitor);
@@ -871,7 +871,7 @@ struct StorageConflictVisitor<'a, 'tcx> {
     // benchmarks for coroutines.
     local_conflicts: BitMatrix<Local, Local>,
     // We keep this bitset as a buffer to avoid reallocating memory.
-    eligible_storage_live: BitSet<Local>,
+    eligible_storage_live: DenseBitSet<Local>,
 }
 
 impl<'a, 'tcx> ResultsVisitor<'a, 'tcx, MaybeRequiresStorage<'a, 'tcx>>
@@ -880,7 +880,7 @@ impl<'a, 'tcx> ResultsVisitor<'a, 'tcx, MaybeRequiresStorage<'a, 'tcx>>
     fn visit_after_early_statement_effect(
         &mut self,
         _results: &mut Results<'tcx, MaybeRequiresStorage<'a, 'tcx>>,
-        state: &BitSet<Local>,
+        state: &DenseBitSet<Local>,
         _statement: &'a Statement<'tcx>,
         loc: Location,
     ) {
@@ -890,7 +890,7 @@ impl<'a, 'tcx> ResultsVisitor<'a, 'tcx, MaybeRequiresStorage<'a, 'tcx>>
     fn visit_after_early_terminator_effect(
         &mut self,
         _results: &mut Results<'tcx, MaybeRequiresStorage<'a, 'tcx>>,
-        state: &BitSet<Local>,
+        state: &DenseBitSet<Local>,
         _terminator: &'a Terminator<'tcx>,
         loc: Location,
     ) {
@@ -899,7 +899,7 @@ impl<'a, 'tcx> ResultsVisitor<'a, 'tcx, MaybeRequiresStorage<'a, 'tcx>>
 }
 
 impl StorageConflictVisitor<'_, '_> {
-    fn apply_state(&mut self, state: &BitSet<Local>, loc: Location) {
+    fn apply_state(&mut self, state: &DenseBitSet<Local>, loc: Location) {
         // Ignore unreachable blocks.
         if let TerminatorKind::Unreachable = self.body.basic_blocks[loc.block].terminator().kind {
             return;
@@ -924,7 +924,7 @@ fn compute_layout<'tcx>(
 ) -> (
     IndexVec<Local, Option<(Ty<'tcx>, VariantIdx, FieldIdx)>>,
     CoroutineLayout<'tcx>,
-    IndexVec<BasicBlock, Option<BitSet<Local>>>,
+    IndexVec<BasicBlock, Option<DenseBitSet<Local>>>,
 ) {
     let LivenessInfo {
         saved_locals,
diff --git a/compiler/rustc_mir_transform/src/coverage/counters.rs b/compiler/rustc_mir_transform/src/coverage/counters.rs
index 9e80f1f1c4a..a9111a5ef9b 100644
--- a/compiler/rustc_mir_transform/src/coverage/counters.rs
+++ b/compiler/rustc_mir_transform/src/coverage/counters.rs
@@ -5,7 +5,7 @@ use rustc_data_structures::captures::Captures;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::graph::DirectedGraph;
 use rustc_index::IndexVec;
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_middle::mir::coverage::{CounterId, CovTerm, Expression, ExpressionId, Op};
 use tracing::{debug, debug_span, instrument};
 
@@ -77,7 +77,7 @@ impl CoverageCounters {
     /// counters or counter expressions for nodes and edges as required.
     pub(super) fn make_bcb_counters(
         graph: &CoverageGraph,
-        bcb_needs_counter: &BitSet<BasicCoverageBlock>,
+        bcb_needs_counter: &DenseBitSet<BasicCoverageBlock>,
     ) -> Self {
         let mut builder = CountersBuilder::new(graph, bcb_needs_counter);
         builder.make_bcb_counters();
@@ -220,13 +220,16 @@ fn sibling_out_edge_targets(
 /// the set of nodes that need counters.
 struct CountersBuilder<'a> {
     graph: &'a CoverageGraph,
-    bcb_needs_counter: &'a BitSet<BasicCoverageBlock>,
+    bcb_needs_counter: &'a DenseBitSet<BasicCoverageBlock>,
 
     site_counters: FxHashMap<Site, SiteCounter>,
 }
 
 impl<'a> CountersBuilder<'a> {
-    fn new(graph: &'a CoverageGraph, bcb_needs_counter: &'a BitSet<BasicCoverageBlock>) -> Self {
+    fn new(
+        graph: &'a CoverageGraph,
+        bcb_needs_counter: &'a DenseBitSet<BasicCoverageBlock>,
+    ) -> Self {
         assert_eq!(graph.num_nodes(), bcb_needs_counter.domain_size());
         Self { graph, bcb_needs_counter, site_counters: FxHashMap::default() }
     }
diff --git a/compiler/rustc_mir_transform/src/coverage/graph.rs b/compiler/rustc_mir_transform/src/coverage/graph.rs
index ad6774fccd6..3fa8b063fa7 100644
--- a/compiler/rustc_mir_transform/src/coverage/graph.rs
+++ b/compiler/rustc_mir_transform/src/coverage/graph.rs
@@ -8,7 +8,7 @@ use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::graph::dominators::Dominators;
 use rustc_data_structures::graph::{self, DirectedGraph, StartNode};
 use rustc_index::IndexVec;
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_middle::mir::{self, BasicBlock, Terminator, TerminatorKind};
 use tracing::debug;
 
@@ -27,7 +27,7 @@ pub(crate) struct CoverageGraph {
     /// their relative order is consistent but arbitrary.
     dominator_order_rank: IndexVec<BasicCoverageBlock, u32>,
     /// A loop header is a node that dominates one or more of its predecessors.
-    is_loop_header: BitSet<BasicCoverageBlock>,
+    is_loop_header: DenseBitSet<BasicCoverageBlock>,
     /// For each node, the loop header node of its nearest enclosing loop.
     /// This forms a linked list that can be traversed to find all enclosing loops.
     enclosing_loop_header: IndexVec<BasicCoverageBlock, Option<BasicCoverageBlock>>,
@@ -72,7 +72,7 @@ impl CoverageGraph {
             predecessors,
             dominators: None,
             dominator_order_rank: IndexVec::from_elem_n(0, num_nodes),
-            is_loop_header: BitSet::new_empty(num_nodes),
+            is_loop_header: DenseBitSet::new_empty(num_nodes),
             enclosing_loop_header: IndexVec::from_elem_n(None, num_nodes),
         };
         assert_eq!(num_nodes, this.num_nodes());
diff --git a/compiler/rustc_mir_transform/src/coverage/mappings.rs b/compiler/rustc_mir_transform/src/coverage/mappings.rs
index 4185b3f4d4d..8d0d92dc367 100644
--- a/compiler/rustc_mir_transform/src/coverage/mappings.rs
+++ b/compiler/rustc_mir_transform/src/coverage/mappings.rs
@@ -3,7 +3,7 @@ use std::collections::BTreeSet;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::graph::DirectedGraph;
 use rustc_index::IndexVec;
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_middle::mir::coverage::{
     BlockMarkerId, BranchSpan, ConditionId, ConditionInfo, CoverageInfoHi, CoverageKind,
 };
@@ -128,7 +128,7 @@ pub(super) fn extract_all_mapping_info_from_mir<'tcx>(
 }
 
 impl ExtractedMappings {
-    pub(super) fn all_bcbs_with_counter_mappings(&self) -> BitSet<BasicCoverageBlock> {
+    pub(super) fn all_bcbs_with_counter_mappings(&self) -> DenseBitSet<BasicCoverageBlock> {
         // Fully destructure self to make sure we don't miss any fields that have mappings.
         let Self {
             num_bcbs,
@@ -140,7 +140,7 @@ impl ExtractedMappings {
         } = self;
 
         // Identify which BCBs have one or more mappings.
-        let mut bcbs_with_counter_mappings = BitSet::new_empty(*num_bcbs);
+        let mut bcbs_with_counter_mappings = DenseBitSet::new_empty(*num_bcbs);
         let mut insert = |bcb| {
             bcbs_with_counter_mappings.insert(bcb);
         };
@@ -172,8 +172,8 @@ impl ExtractedMappings {
     }
 
     /// Returns the set of BCBs that have one or more `Code` mappings.
-    pub(super) fn bcbs_with_ordinary_code_mappings(&self) -> BitSet<BasicCoverageBlock> {
-        let mut bcbs = BitSet::new_empty(self.num_bcbs);
+    pub(super) fn bcbs_with_ordinary_code_mappings(&self) -> DenseBitSet<BasicCoverageBlock> {
+        let mut bcbs = DenseBitSet::new_empty(self.num_bcbs);
         for &CodeMapping { span: _, bcb } in &self.code_mappings {
             bcbs.insert(bcb);
         }
@@ -367,9 +367,8 @@ fn calc_test_vectors_index(conditions: &mut Vec<MCDCBranch>) -> usize {
         })
         .collect::<FxIndexMap<_, _>>();
 
-    let mut queue = std::collections::VecDeque::from_iter(
-        next_conditions.swap_remove(&ConditionId::START).into_iter(),
-    );
+    let mut queue =
+        std::collections::VecDeque::from_iter(next_conditions.swap_remove(&ConditionId::START));
     num_paths_stats[ConditionId::START] = 1;
     let mut decision_end_nodes = Vec::new();
     while let Some(branch) = queue.pop_front() {
diff --git a/compiler/rustc_mir_transform/src/coverage/query.rs b/compiler/rustc_mir_transform/src/coverage/query.rs
index edaec3c7965..3e7cf8541c2 100644
--- a/compiler/rustc_mir_transform/src/coverage/query.rs
+++ b/compiler/rustc_mir_transform/src/coverage/query.rs
@@ -1,5 +1,5 @@
 use rustc_data_structures::captures::Captures;
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::coverage::{
     CounterId, CovTerm, CoverageIdsInfo, CoverageKind, Expression, ExpressionId,
@@ -92,13 +92,13 @@ fn coverage_ids_info<'tcx>(
 
     let Some(fn_cov_info) = mir_body.function_coverage_info.as_deref() else {
         return CoverageIdsInfo {
-            counters_seen: BitSet::new_empty(0),
-            zero_expressions: BitSet::new_empty(0),
+            counters_seen: DenseBitSet::new_empty(0),
+            zero_expressions: DenseBitSet::new_empty(0),
         };
     };
 
-    let mut counters_seen = BitSet::new_empty(fn_cov_info.num_counters);
-    let mut expressions_seen = BitSet::new_filled(fn_cov_info.expressions.len());
+    let mut counters_seen = DenseBitSet::new_empty(fn_cov_info.num_counters);
+    let mut expressions_seen = DenseBitSet::new_filled(fn_cov_info.expressions.len());
 
     // For each expression ID that is directly used by one or more mappings,
     // mark it as not-yet-seen. This indicates that we expect to see a
@@ -148,23 +148,23 @@ fn is_inlined(body: &Body<'_>, statement: &Statement<'_>) -> bool {
     scope_data.inlined.is_some() || scope_data.inlined_parent_scope.is_some()
 }
 
-/// Identify expressions that will always have a value of zero, and note
-/// their IDs in a `BitSet`. Mappings that refer to a zero expression
-/// can instead become mappings to a constant zero value.
+/// Identify expressions that will always have a value of zero, and note their
+/// IDs in a `DenseBitSet`. Mappings that refer to a zero expression can instead
+/// become mappings to a constant zero value.
 ///
 /// This function mainly exists to preserve the simplifications that were
 /// already being performed by the Rust-side expression renumbering, so that
 /// the resulting coverage mappings don't get worse.
 fn identify_zero_expressions(
     fn_cov_info: &FunctionCoverageInfo,
-    counters_seen: &BitSet<CounterId>,
-    expressions_seen: &BitSet<ExpressionId>,
-) -> BitSet<ExpressionId> {
+    counters_seen: &DenseBitSet<CounterId>,
+    expressions_seen: &DenseBitSet<ExpressionId>,
+) -> DenseBitSet<ExpressionId> {
     // The set of expressions that either were optimized out entirely, or
     // have zero as both of their operands, and will therefore always have
     // a value of zero. Other expressions that refer to these as operands
     // can have those operands replaced with `CovTerm::Zero`.
-    let mut zero_expressions = BitSet::new_empty(fn_cov_info.expressions.len());
+    let mut zero_expressions = DenseBitSet::new_empty(fn_cov_info.expressions.len());
 
     // Simplify a copy of each expression based on lower-numbered expressions,
     // and then update the set of always-zero expressions if necessary.
@@ -228,8 +228,8 @@ fn identify_zero_expressions(
 /// into account knowledge of which counters are unused and which expressions
 /// are always zero.
 fn is_zero_term(
-    counters_seen: &BitSet<CounterId>,
-    zero_expressions: &BitSet<ExpressionId>,
+    counters_seen: &DenseBitSet<CounterId>,
+    zero_expressions: &DenseBitSet<ExpressionId>,
     term: CovTerm,
 ) -> bool {
     match term {
diff --git a/compiler/rustc_mir_transform/src/cross_crate_inline.rs b/compiler/rustc_mir_transform/src/cross_crate_inline.rs
index e1f1dd83f0d..8fce856687c 100644
--- a/compiler/rustc_mir_transform/src/cross_crate_inline.rs
+++ b/compiler/rustc_mir_transform/src/cross_crate_inline.rs
@@ -46,7 +46,7 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     // #[inline(never)] to force code generation.
     match codegen_fn_attrs.inline {
         InlineAttr::Never => return false,
-        InlineAttr::Hint | InlineAttr::Always => return true,
+        InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. } => return true,
         _ => {}
     }
 
@@ -69,8 +69,9 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     // Don't do any inference if codegen optimizations are disabled and also MIR inlining is not
     // enabled. This ensures that we do inference even if someone only passes -Zinline-mir,
     // which is less confusing than having to also enable -Copt-level=1.
-    if matches!(tcx.sess.opts.optimize, OptLevel::No) && !pm::should_run_pass(tcx, &inline::Inline)
-    {
+    let inliner_will_run = pm::should_run_pass(tcx, &inline::Inline)
+        || inline::ForceInline::should_run_pass_for_callee(tcx, def_id.to_def_id());
+    if matches!(tcx.sess.opts.optimize, OptLevel::No) && !inliner_will_run {
         return false;
     }
 
diff --git a/compiler/rustc_mir_transform/src/dead_store_elimination.rs b/compiler/rustc_mir_transform/src/dead_store_elimination.rs
index 0c75cdadc92..434e921d439 100644
--- a/compiler/rustc_mir_transform/src/dead_store_elimination.rs
+++ b/compiler/rustc_mir_transform/src/dead_store_elimination.rs
@@ -26,8 +26,8 @@ use crate::util::is_within_packed;
 
 /// Performs the optimization on the body
 ///
-/// The `borrowed` set must be a `BitSet` of all the locals that are ever borrowed in this body. It
-/// can be generated via the [`borrowed_locals`] function.
+/// The `borrowed` set must be a `DenseBitSet` of all the locals that are ever borrowed in this
+/// body. It can be generated via the [`borrowed_locals`] function.
 fn eliminate<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
     let borrowed_locals = borrowed_locals(body);
 
diff --git a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
index 67b215c7c9d..049f13ce96d 100644
--- a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
+++ b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
@@ -6,7 +6,7 @@
 //! dependent crates can use them.
 
 use rustc_hir::def_id::LocalDefId;
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_middle::mir::visit::{NonMutatingUseContext, PlaceContext, Visitor};
 use rustc_middle::mir::{Body, Location, Operand, Place, RETURN_PLACE, Terminator, TerminatorKind};
 use rustc_middle::ty::{self, DeducedParamAttrs, Ty, TyCtxt};
@@ -18,13 +18,13 @@ struct DeduceReadOnly {
     /// Each bit is indexed by argument number, starting at zero (so 0 corresponds to local decl
     /// 1). The bit is true if the argument may have been mutated or false if we know it hasn't
     /// been up to the point we're at.
-    mutable_args: BitSet<usize>,
+    mutable_args: DenseBitSet<usize>,
 }
 
 impl DeduceReadOnly {
     /// Returns a new DeduceReadOnly instance.
     fn new(arg_count: usize) -> Self {
-        Self { mutable_args: BitSet::new_empty(arg_count) }
+        Self { mutable_args: DenseBitSet::new_empty(arg_count) }
     }
 }
 
diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs
index e99bee6a01f..1ac4d835946 100644
--- a/compiler/rustc_mir_transform/src/dest_prop.rs
+++ b/compiler/rustc_mir_transform/src/dest_prop.rs
@@ -132,7 +132,7 @@
 //! [attempt 3]: https://github.com/rust-lang/rust/pull/72632
 
 use rustc_data_structures::fx::{FxIndexMap, IndexEntry, IndexOccupiedEntry};
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_index::interval::SparseIntervalMatrix;
 use rustc_middle::bug;
 use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor};
@@ -204,7 +204,8 @@ impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation {
             // Because we only filter once per round, it is unsound to use a local for more than
             // one merge operation within a single round of optimizations. We store here which ones
             // we have already used.
-            let mut merged_locals: BitSet<Local> = BitSet::new_empty(body.local_decls.len());
+            let mut merged_locals: DenseBitSet<Local> =
+                DenseBitSet::new_empty(body.local_decls.len());
 
             // This is the set of merges we will apply this round. It is a subset of the candidates.
             let mut merges = FxIndexMap::default();
@@ -274,7 +275,7 @@ fn apply_merges<'tcx>(
     body: &mut Body<'tcx>,
     tcx: TyCtxt<'tcx>,
     merges: FxIndexMap<Local, Local>,
-    merged_locals: BitSet<Local>,
+    merged_locals: DenseBitSet<Local>,
 ) {
     let mut merger = Merger { tcx, merges, merged_locals };
     merger.visit_body_preserves_cfg(body);
@@ -283,7 +284,7 @@ fn apply_merges<'tcx>(
 struct Merger<'tcx> {
     tcx: TyCtxt<'tcx>,
     merges: FxIndexMap<Local, Local>,
-    merged_locals: BitSet<Local>,
+    merged_locals: DenseBitSet<Local>,
 }
 
 impl<'tcx> MutVisitor<'tcx> for Merger<'tcx> {
@@ -351,7 +352,7 @@ impl Candidates {
     /// Collects the candidates for merging.
     ///
     /// This is responsible for enforcing the first and third bullet point.
-    fn reset_and_find<'tcx>(&mut self, body: &Body<'tcx>, borrowed: &BitSet<Local>) {
+    fn reset_and_find<'tcx>(&mut self, body: &Body<'tcx>, borrowed: &DenseBitSet<Local>) {
         self.c.clear();
         self.reverse.clear();
         let mut visitor = FindAssignments { body, candidates: &mut self.c, borrowed };
@@ -735,7 +736,7 @@ fn places_to_candidate_pair<'tcx>(
 struct FindAssignments<'a, 'tcx> {
     body: &'a Body<'tcx>,
     candidates: &'a mut FxIndexMap<Local, Vec<Local>>,
-    borrowed: &'a BitSet<Local>,
+    borrowed: &'a DenseBitSet<Local>,
 }
 
 impl<'tcx> Visitor<'tcx> for FindAssignments<'_, 'tcx> {
diff --git a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs
index b909dfa1320..d6ecadbfe29 100644
--- a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs
+++ b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs
@@ -29,13 +29,8 @@ fn build_ptr_tys<'tcx>(
 pub(super) fn build_projection<'tcx>(
     unique_ty: Ty<'tcx>,
     nonnull_ty: Ty<'tcx>,
-    ptr_ty: Ty<'tcx>,
-) -> [PlaceElem<'tcx>; 3] {
-    [
-        PlaceElem::Field(FieldIdx::ZERO, unique_ty),
-        PlaceElem::Field(FieldIdx::ZERO, nonnull_ty),
-        PlaceElem::Field(FieldIdx::ZERO, ptr_ty),
-    ]
+) -> [PlaceElem<'tcx>; 2] {
+    [PlaceElem::Field(FieldIdx::ZERO, unique_ty), PlaceElem::Field(FieldIdx::ZERO, nonnull_ty)]
 }
 
 struct ElaborateBoxDerefVisitor<'a, 'tcx> {
@@ -75,10 +70,14 @@ impl<'a, 'tcx> MutVisitor<'tcx> for ElaborateBoxDerefVisitor<'a, 'tcx> {
             self.patch.add_assign(
                 location,
                 Place::from(ptr_local),
-                Rvalue::Use(Operand::Copy(
-                    Place::from(place.local)
-                        .project_deeper(&build_projection(unique_ty, nonnull_ty, ptr_ty), tcx),
-                )),
+                Rvalue::Cast(
+                    CastKind::Transmute,
+                    Operand::Copy(
+                        Place::from(place.local)
+                            .project_deeper(&build_projection(unique_ty, nonnull_ty), tcx),
+                    ),
+                    ptr_ty,
+                ),
             );
 
             place.local = ptr_local;
@@ -133,8 +132,10 @@ impl<'tcx> crate::MirPass<'tcx> for ElaborateBoxDerefs {
                         let (unique_ty, nonnull_ty, ptr_ty) =
                             build_ptr_tys(tcx, boxed_ty, unique_did, nonnull_did);
 
-                        new_projections
-                            .extend_from_slice(&build_projection(unique_ty, nonnull_ty, ptr_ty));
+                        new_projections.extend_from_slice(&build_projection(unique_ty, nonnull_ty));
+                        // While we can't project into `NonNull<_>` in a basic block
+                        // due to MCP#807, this is debug info where it's fine.
+                        new_projections.push(PlaceElem::Field(FieldIdx::ZERO, ptr_ty));
                         new_projections.push(PlaceElem::Deref);
                     } else if let Some(new_projections) = new_projections.as_mut() {
                         // Keep building up our projections list once we've started it.
diff --git a/compiler/rustc_mir_transform/src/elaborate_drops.rs b/compiler/rustc_mir_transform/src/elaborate_drops.rs
index 3ebc9113725..988f1a25561 100644
--- a/compiler/rustc_mir_transform/src/elaborate_drops.rs
+++ b/compiler/rustc_mir_transform/src/elaborate_drops.rs
@@ -2,7 +2,7 @@ use std::fmt;
 
 use rustc_abi::{FieldIdx, VariantIdx};
 use rustc_index::IndexVec;
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_middle::mir::patch::MirPatch;
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, TyCtxt};
@@ -96,10 +96,10 @@ impl<'tcx> crate::MirPass<'tcx> for ElaborateDrops {
 fn compute_dead_unwinds<'a, 'tcx>(
     body: &'a Body<'tcx>,
     flow_inits: &mut ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>>,
-) -> BitSet<BasicBlock> {
+) -> DenseBitSet<BasicBlock> {
     // We only need to do this pass once, because unwind edges can only
     // reach cleanup blocks, which can't have unwind edges themselves.
-    let mut dead_unwinds = BitSet::new_empty(body.basic_blocks.len());
+    let mut dead_unwinds = DenseBitSet::new_empty(body.basic_blocks.len());
     for (bb, bb_data) in body.basic_blocks.iter_enumerated() {
         let TerminatorKind::Drop { place, unwind: UnwindAction::Cleanup(_), .. } =
             bb_data.terminator().kind
diff --git a/compiler/rustc_mir_transform/src/errors.rs b/compiler/rustc_mir_transform/src/errors.rs
index 2d9eeddea2e..015633d145f 100644
--- a/compiler/rustc_mir_transform/src/errors.rs
+++ b/compiler/rustc_mir_transform/src/errors.rs
@@ -4,8 +4,8 @@ use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic};
 use rustc_middle::mir::AssertKind;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::lint::{self, Lint};
-use rustc_span::Span;
 use rustc_span::def_id::DefId;
+use rustc_span::{Span, Symbol};
 
 use crate::fluent_generated as fluent;
 
@@ -142,3 +142,29 @@ pub(crate) struct MustNotSuspendReason {
 #[note(mir_transform_note2)]
 #[help]
 pub(crate) struct UndefinedTransmute;
+
+#[derive(Diagnostic)]
+#[diag(mir_transform_force_inline)]
+#[note]
+pub(crate) struct ForceInlineFailure {
+    #[label(mir_transform_caller)]
+    pub caller_span: Span,
+    #[label(mir_transform_callee)]
+    pub callee_span: Span,
+    #[label(mir_transform_attr)]
+    pub attr_span: Span,
+    #[primary_span]
+    #[label(mir_transform_call)]
+    pub call_span: Span,
+    pub callee: String,
+    pub caller: String,
+    pub reason: &'static str,
+    #[subdiagnostic]
+    pub justification: Option<ForceInlineJustification>,
+}
+
+#[derive(Subdiagnostic)]
+#[note(mir_transform_force_inline_justification)]
+pub(crate) struct ForceInlineJustification {
+    pub sym: Symbol,
+}
diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index 283ed94b615..affc4cf0afc 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -94,7 +94,7 @@ use rustc_const_eval::interpret::{
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_data_structures::graph::dominators::Dominators;
 use rustc_hir::def::DefKind;
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_index::{IndexVec, newtype_index};
 use rustc_middle::bug;
 use rustc_middle::mir::interpret::GlobalAlloc;
@@ -256,7 +256,7 @@ struct VnState<'body, 'tcx> {
     feature_unsized_locals: bool,
     ssa: &'body SsaLocals,
     dominators: Dominators<BasicBlock>,
-    reused_locals: BitSet<Local>,
+    reused_locals: DenseBitSet<Local>,
 }
 
 impl<'body, 'tcx> VnState<'body, 'tcx> {
@@ -287,7 +287,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             feature_unsized_locals: tcx.features().unsized_locals(),
             ssa,
             dominators,
-            reused_locals: BitSet::new_empty(local_decls.len()),
+            reused_locals: DenseBitSet::new_empty(local_decls.len()),
         }
     }
 
@@ -1366,57 +1366,108 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             return self.new_opaque();
         }
 
-        let mut was_updated = false;
+        let mut was_ever_updated = false;
+        loop {
+            let mut was_updated_this_iteration = false;
+
+            // Transmuting between raw pointers is just a pointer cast so long as
+            // they have the same metadata type (like `*const i32` <=> `*mut u64`
+            // or `*mut [i32]` <=> `*const [u64]`), including the common special
+            // case of `*const T` <=> `*mut T`.
+            if let Transmute = kind
+                && from.is_unsafe_ptr()
+                && to.is_unsafe_ptr()
+                && self.pointers_have_same_metadata(from, to)
+            {
+                *kind = PtrToPtr;
+                was_updated_this_iteration = true;
+            }
 
-        // If that cast just casts away the metadata again,
-        if let PtrToPtr = kind
-            && let Value::Aggregate(AggregateTy::RawPtr { data_pointer_ty, .. }, _, fields) =
-                self.get(value)
-            && let ty::RawPtr(to_pointee, _) = to.kind()
-            && to_pointee.is_sized(self.tcx, self.typing_env())
-        {
-            from = *data_pointer_ty;
-            value = fields[0];
-            was_updated = true;
-            if *data_pointer_ty == to {
-                return Some(fields[0]);
+            // If a cast just casts away the metadata again, then we can get it by
+            // casting the original thin pointer passed to `from_raw_parts`
+            if let PtrToPtr = kind
+                && let Value::Aggregate(AggregateTy::RawPtr { data_pointer_ty, .. }, _, fields) =
+                    self.get(value)
+                && let ty::RawPtr(to_pointee, _) = to.kind()
+                && to_pointee.is_sized(self.tcx, self.typing_env())
+            {
+                from = *data_pointer_ty;
+                value = fields[0];
+                was_updated_this_iteration = true;
+                if *data_pointer_ty == to {
+                    return Some(fields[0]);
+                }
             }
-        }
 
-        // PtrToPtr-then-PtrToPtr can skip the intermediate step
-        if let PtrToPtr = kind
-            && let Value::Cast { kind: inner_kind, value: inner_value, from: inner_from, to: _ } =
-                *self.get(value)
-            && let PtrToPtr = inner_kind
-        {
-            from = inner_from;
-            value = inner_value;
-            was_updated = true;
-            if inner_from == to {
-                return Some(inner_value);
+            // Aggregate-then-Transmute can just transmute the original field value,
+            // so long as the bytes of a value from only from a single field.
+            if let Transmute = kind
+                && let Value::Aggregate(_aggregate_ty, variant_idx, field_values) = self.get(value)
+                && let Some((field_idx, field_ty)) =
+                    self.value_is_all_in_one_field(from, *variant_idx)
+            {
+                from = field_ty;
+                value = field_values[field_idx.as_usize()];
+                was_updated_this_iteration = true;
+                if field_ty == to {
+                    return Some(value);
+                }
             }
-        }
 
-        // PtrToPtr-then-Transmute can just transmute the original, so long as the
-        // PtrToPtr didn't change metadata (and thus the size of the pointer)
-        if let Transmute = kind
-            && let Value::Cast {
-                kind: PtrToPtr,
+            // Various cast-then-cast cases can be simplified.
+            if let Value::Cast {
+                kind: inner_kind,
                 value: inner_value,
                 from: inner_from,
                 to: inner_to,
             } = *self.get(value)
-            && self.pointers_have_same_metadata(inner_from, inner_to)
-        {
-            from = inner_from;
-            value = inner_value;
-            was_updated = true;
-            if inner_from == to {
-                return Some(inner_value);
+            {
+                let new_kind = match (inner_kind, *kind) {
+                    // Even if there's a narrowing cast in here that's fine, because
+                    // things like `*mut [i32] -> *mut i32 -> *const i32` and
+                    // `*mut [i32] -> *const [i32] -> *const i32` can skip the middle in MIR.
+                    (PtrToPtr, PtrToPtr) => Some(PtrToPtr),
+                    // PtrToPtr-then-Transmute is fine so long as the pointer cast is identity:
+                    // `*const T -> *mut T -> NonNull<T>` is fine, but we need to check for narrowing
+                    // to skip things like `*const [i32] -> *const i32 -> NonNull<T>`.
+                    (PtrToPtr, Transmute)
+                        if self.pointers_have_same_metadata(inner_from, inner_to) =>
+                    {
+                        Some(Transmute)
+                    }
+                    // Similarly, for Transmute-then-PtrToPtr. Note that we need to check different
+                    // variables for their metadata, and thus this can't merge with the previous arm.
+                    (Transmute, PtrToPtr) if self.pointers_have_same_metadata(from, to) => {
+                        Some(Transmute)
+                    }
+                    // If would be legal to always do this, but we don't want to hide information
+                    // from the backend that it'd otherwise be able to use for optimizations.
+                    (Transmute, Transmute)
+                        if !self.type_may_have_niche_of_interest_to_backend(inner_to) =>
+                    {
+                        Some(Transmute)
+                    }
+                    _ => None,
+                };
+                if let Some(new_kind) = new_kind {
+                    *kind = new_kind;
+                    from = inner_from;
+                    value = inner_value;
+                    was_updated_this_iteration = true;
+                    if inner_from == to {
+                        return Some(inner_value);
+                    }
+                }
+            }
+
+            if was_updated_this_iteration {
+                was_ever_updated = true;
+            } else {
+                break;
             }
         }
 
-        if was_updated && let Some(op) = self.try_as_operand(value, location) {
+        if was_ever_updated && let Some(op) = self.try_as_operand(value, location) {
             *operand = op;
         }
 
@@ -1438,6 +1489,54 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             false
         }
     }
+
+    /// Returns `false` if we know for sure that this type has no interesting niche,
+    /// and thus we can skip transmuting through it without worrying.
+    ///
+    /// The backend will emit `assume`s when transmuting between types with niches,
+    /// so we want to preserve `i32 -> char -> u32` so that that data is around,
+    /// but it's fine to skip whole-range-is-value steps like `A -> u32 -> B`.
+    fn type_may_have_niche_of_interest_to_backend(&self, ty: Ty<'tcx>) -> bool {
+        let Ok(layout) = self.ecx.layout_of(ty) else {
+            // If it's too generic or something, then assume it might be interesting later.
+            return true;
+        };
+
+        match layout.backend_repr {
+            BackendRepr::Uninhabited => true,
+            BackendRepr::Scalar(a) => !a.is_always_valid(&self.ecx),
+            BackendRepr::ScalarPair(a, b) => {
+                !a.is_always_valid(&self.ecx) || !b.is_always_valid(&self.ecx)
+            }
+            BackendRepr::Vector { .. } | BackendRepr::Memory { .. } => false,
+        }
+    }
+
+    fn value_is_all_in_one_field(
+        &self,
+        ty: Ty<'tcx>,
+        variant: VariantIdx,
+    ) -> Option<(FieldIdx, Ty<'tcx>)> {
+        if let Ok(layout) = self.ecx.layout_of(ty)
+            && let abi::Variants::Single { index } = layout.variants
+            && index == variant
+            && let Some((field_idx, field_layout)) = layout.non_1zst_field(&self.ecx)
+            && layout.size == field_layout.size
+        {
+            // We needed to check the variant to avoid trying to read the tag
+            // field from an enum where no fields have variants, since that tag
+            // field isn't in the `Aggregate` from which we're getting values.
+            Some((FieldIdx::from_usize(field_idx), field_layout.ty))
+        } else if let ty::Adt(adt, args) = ty.kind()
+            && adt.is_struct()
+            && adt.repr().transparent()
+            && let [single_field] = adt.non_enum_variant().fields.raw.as_slice()
+        {
+            Some((FieldIdx::ZERO, single_field.ty(self.tcx, args)))
+        } else {
+            None
+        }
+    }
 }
 
 fn op_to_prop_const<'tcx>(
@@ -1615,7 +1714,7 @@ impl<'tcx> MutVisitor<'tcx> for VnState<'_, 'tcx> {
 
 struct StorageRemover<'tcx> {
     tcx: TyCtxt<'tcx>,
-    reused_locals: BitSet<Local>,
+    reused_locals: DenseBitSet<Local>,
 }
 
 impl<'tcx> MutVisitor<'tcx> for StorageRemover<'tcx> {
diff --git a/compiler/rustc_mir_transform/src/impossible_predicates.rs b/compiler/rustc_mir_transform/src/impossible_predicates.rs
new file mode 100644
index 00000000000..ba8389bbe2f
--- /dev/null
+++ b/compiler/rustc_mir_transform/src/impossible_predicates.rs
@@ -0,0 +1,56 @@
+//! Check if it's even possible to satisfy the 'where' clauses
+//! for this item.
+//!
+//! It's possible to `#!feature(trivial_bounds)]` to write
+//! a function with impossible to satisfy clauses, e.g.:
+//! `fn foo() where String: Copy {}`.
+//!
+//! We don't usually need to worry about this kind of case,
+//! since we would get a compilation error if the user tried
+//! to call it. However, since we optimize even without any
+//! calls to the function, we need to make sure that it even
+//! makes sense to try to evaluate the body.
+//!
+//! If there are unsatisfiable where clauses, then all bets are
+//! off, and we just give up.
+//!
+//! We manually filter the predicates, skipping anything that's not
+//! "global". We are in a potentially generic context
+//! (e.g. we are evaluating a function without instantiating generic
+//! parameters, so this filtering serves two purposes:
+//!
+//! 1. We skip evaluating any predicates that we would
+//! never be able prove are unsatisfiable (e.g. `<T as Foo>`
+//! 2. We avoid trying to normalize predicates involving generic
+//! parameters (e.g. `<T as Foo>::MyItem`). This can confuse
+//! the normalization code (leading to cycle errors), since
+//! it's usually never invoked in this way.
+
+use rustc_middle::mir::{Body, START_BLOCK, TerminatorKind};
+use rustc_middle::ty::{TyCtxt, TypeVisitableExt};
+use rustc_trait_selection::traits;
+use tracing::trace;
+
+use crate::pass_manager::MirPass;
+
+pub(crate) struct ImpossiblePredicates;
+
+impl<'tcx> MirPass<'tcx> for ImpossiblePredicates {
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+        let predicates = tcx
+            .predicates_of(body.source.def_id())
+            .predicates
+            .iter()
+            .filter_map(|(p, _)| if p.is_global() { Some(*p) } else { None });
+        if traits::impossible_predicates(tcx, traits::elaborate(tcx, predicates).collect()) {
+            trace!("found unsatisfiable predicates for {:?}", body.source);
+            // Clear the body to only contain a single `unreachable` statement.
+            let bbs = body.basic_blocks.as_mut();
+            bbs.raw.truncate(1);
+            bbs[START_BLOCK].statements.clear();
+            bbs[START_BLOCK].terminator_mut().kind = TerminatorKind::Unreachable;
+            body.var_debug_info.clear();
+            body.local_decls.raw.truncate(body.arg_count + 1);
+        }
+    }
+}
diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs
index 339acbad6b9..470393c9ae1 100644
--- a/compiler/rustc_mir_transform/src/inline.rs
+++ b/compiler/rustc_mir_transform/src/inline.rs
@@ -8,15 +8,14 @@ use rustc_attr_parsing::InlineAttr;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
 use rustc_index::Idx;
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_middle::bug;
-use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
+use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;
 use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, Instance, InstanceKind, Ty, TyCtxt, TypeFlags, TypeVisitableExt};
 use rustc_session::config::{DebugInfo, OptLevel};
 use rustc_span::source_map::Spanned;
-use rustc_span::sym;
 use tracing::{debug, instrument, trace, trace_span};
 
 use crate::cost_checker::CostChecker;
@@ -29,10 +28,6 @@ pub(crate) mod cycle;
 
 const TOP_DOWN_DEPTH_LIMIT: usize = 5;
 
-// Made public so that `mir_drops_elaborated_and_const_checked` can be overridden
-// by custom rustc drivers, running all the steps by themselves. See #114628.
-pub struct Inline;
-
 #[derive(Clone, Debug)]
 struct CallSite<'tcx> {
     callee: Instance<'tcx>,
@@ -41,14 +36,12 @@ struct CallSite<'tcx> {
     source_info: SourceInfo,
 }
 
+// Made public so that `mir_drops_elaborated_and_const_checked` can be overridden
+// by custom rustc drivers, running all the steps by themselves. See #114628.
+pub struct Inline;
+
 impl<'tcx> crate::MirPass<'tcx> for Inline {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
-        // FIXME(#127234): Coverage instrumentation currently doesn't handle inlined
-        // MIR correctly when Modified Condition/Decision Coverage is enabled.
-        if sess.instrument_coverage_mcdc() {
-            return false;
-        }
-
         if let Some(enabled) = sess.opts.unstable_opts.inline_mir {
             return enabled;
         }
@@ -67,7 +60,7 @@ impl<'tcx> crate::MirPass<'tcx> for Inline {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         let span = trace_span!("inline", body = %tcx.def_path_str(body.source.def_id()));
         let _guard = span.enter();
-        if inline(tcx, body) {
+        if inline::<NormalInliner<'tcx>>(tcx, body) {
             debug!("running simplify cfg on {:?}", body.source);
             simplify_cfg(body);
             deref_finder(tcx, body);
@@ -75,47 +68,83 @@ impl<'tcx> crate::MirPass<'tcx> for Inline {
     }
 }
 
-fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
-    let def_id = body.source.def_id().expect_local();
+pub struct ForceInline;
 
-    // Only do inlining into fn bodies.
-    if !tcx.hir().body_owner_kind(def_id).is_fn_or_closure() {
-        return false;
+impl ForceInline {
+    pub fn should_run_pass_for_callee<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool {
+        matches!(tcx.codegen_fn_attrs(def_id).inline, InlineAttr::Force { .. })
     }
-    if body.source.promoted.is_some() {
-        return false;
+}
+
+impl<'tcx> crate::MirPass<'tcx> for ForceInline {
+    fn is_enabled(&self, _: &rustc_session::Session) -> bool {
+        true
     }
-    // Avoid inlining into coroutines, since their `optimized_mir` is used for layout computation,
-    // which can create a cycle, even when no attempt is made to inline the function in the other
-    // direction.
-    if body.coroutine.is_some() {
-        return false;
+
+    fn can_be_overridden(&self) -> bool {
+        false
+    }
+
+    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+        let span = trace_span!("force_inline", body = %tcx.def_path_str(body.source.def_id()));
+        let _guard = span.enter();
+        if inline::<ForceInliner<'tcx>>(tcx, body) {
+            debug!("running simplify cfg on {:?}", body.source);
+            simplify_cfg(body);
+            deref_finder(tcx, body);
+        }
     }
+}
 
-    let typing_env = body.typing_env(tcx);
-    let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id);
+trait Inliner<'tcx> {
+    fn new(tcx: TyCtxt<'tcx>, def_id: DefId, body: &Body<'tcx>) -> Self;
 
-    let mut this = Inliner {
-        tcx,
-        typing_env,
-        codegen_fn_attrs,
-        history: Vec::new(),
-        changed: false,
-        caller_is_inline_forwarder: matches!(
-            codegen_fn_attrs.inline,
-            InlineAttr::Hint | InlineAttr::Always
-        ) && body_is_forwarder(body),
-    };
-    let blocks = START_BLOCK..body.basic_blocks.next_index();
-    this.process_blocks(body, blocks);
-    this.changed
+    fn tcx(&self) -> TyCtxt<'tcx>;
+    fn typing_env(&self) -> ty::TypingEnv<'tcx>;
+    fn history(&self) -> &[DefId];
+    fn caller_def_id(&self) -> DefId;
+
+    /// Has the caller body been changed?
+    fn changed(self) -> bool;
+
+    /// Should inlining happen for a given callee?
+    fn should_inline_for_callee(&self, def_id: DefId) -> bool;
+
+    fn check_caller_mir_body(&self, body: &Body<'tcx>) -> bool;
+
+    /// Returns inlining decision that is based on the examination of callee MIR body.
+    /// Assumes that codegen attributes have been checked for compatibility already.
+    fn check_callee_mir_body(
+        &self,
+        callsite: &CallSite<'tcx>,
+        callee_body: &Body<'tcx>,
+        callee_attrs: &CodegenFnAttrs,
+    ) -> Result<(), &'static str>;
+
+    // How many callsites in a body are we allowed to inline? We need to limit this in order
+    // to prevent super-linear growth in MIR size.
+    fn inline_limit_for_block(&self) -> Option<usize>;
+
+    /// Called when inlining succeeds.
+    fn on_inline_success(
+        &mut self,
+        callsite: &CallSite<'tcx>,
+        caller_body: &mut Body<'tcx>,
+        new_blocks: std::ops::Range<BasicBlock>,
+    );
+
+    /// Called when inlining failed or was not performed.
+    fn on_inline_failure(&self, callsite: &CallSite<'tcx>, reason: &'static str);
+
+    /// Called when the inline limit for a body is reached.
+    fn on_inline_limit_reached(&self) -> bool;
 }
 
-struct Inliner<'tcx> {
+struct ForceInliner<'tcx> {
     tcx: TyCtxt<'tcx>,
     typing_env: ty::TypingEnv<'tcx>,
-    /// Caller codegen attributes.
-    codegen_fn_attrs: &'tcx CodegenFnAttrs,
+    /// `DefId` of caller.
+    def_id: DefId,
     /// Stack of inlined instances.
     /// We only check the `DefId` and not the args because we want to
     /// avoid inlining cases of polymorphic recursion.
@@ -124,366 +153,203 @@ struct Inliner<'tcx> {
     history: Vec<DefId>,
     /// Indicates that the caller body has been modified.
     changed: bool,
-    /// Indicates that the caller is #[inline] and just calls another function,
-    /// and thus we can inline less into it as it'll be inlined itself.
-    caller_is_inline_forwarder: bool,
 }
 
-impl<'tcx> Inliner<'tcx> {
-    fn process_blocks(&mut self, caller_body: &mut Body<'tcx>, blocks: Range<BasicBlock>) {
-        // How many callsites in this body are we allowed to inline? We need to limit this in order
-        // to prevent super-linear growth in MIR size
-        let inline_limit = match self.history.len() {
-            0 => usize::MAX,
-            1..=TOP_DOWN_DEPTH_LIMIT => 1,
-            _ => return,
-        };
-        let mut inlined_count = 0;
-        for bb in blocks {
-            let bb_data = &caller_body[bb];
-            if bb_data.is_cleanup {
-                continue;
-            }
-
-            let Some(callsite) = self.resolve_callsite(caller_body, bb, bb_data) else {
-                continue;
-            };
-
-            let span = trace_span!("process_blocks", %callsite.callee, ?bb);
-            let _guard = span.enter();
-
-            match self.try_inlining(caller_body, &callsite) {
-                Err(reason) => {
-                    debug!("not-inlined {} [{}]", callsite.callee, reason);
-                }
-                Ok(new_blocks) => {
-                    debug!("inlined {}", callsite.callee);
-                    self.changed = true;
-
-                    self.history.push(callsite.callee.def_id());
-                    self.process_blocks(caller_body, new_blocks);
-                    self.history.pop();
-
-                    inlined_count += 1;
-                    if inlined_count == inline_limit {
-                        debug!("inline count reached");
-                        return;
-                    }
-                }
-            }
-        }
+impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {
+    fn new(tcx: TyCtxt<'tcx>, def_id: DefId, body: &Body<'tcx>) -> Self {
+        Self { tcx, typing_env: body.typing_env(tcx), def_id, history: Vec::new(), changed: false }
     }
 
-    /// Attempts to inline a callsite into the caller body. When successful returns basic blocks
-    /// containing the inlined body. Otherwise returns an error describing why inlining didn't take
-    /// place.
-    fn try_inlining(
-        &self,
-        caller_body: &mut Body<'tcx>,
-        callsite: &CallSite<'tcx>,
-    ) -> Result<std::ops::Range<BasicBlock>, &'static str> {
-        self.check_mir_is_available(caller_body, callsite.callee)?;
-
-        let callee_attrs = self.tcx.codegen_fn_attrs(callsite.callee.def_id());
-        let cross_crate_inlinable = self.tcx.cross_crate_inlinable(callsite.callee.def_id());
-        self.check_codegen_attributes(callsite, callee_attrs, cross_crate_inlinable)?;
-
-        // Intrinsic fallback bodies are automatically made cross-crate inlineable,
-        // but at this stage we don't know whether codegen knows the intrinsic,
-        // so just conservatively don't inline it. This also ensures that we do not
-        // accidentally inline the body of an intrinsic that *must* be overridden.
-        if self.tcx.has_attr(callsite.callee.def_id(), sym::rustc_intrinsic) {
-            return Err("Callee is an intrinsic, do not inline fallback bodies");
-        }
-
-        let terminator = caller_body[callsite.block].terminator.as_ref().unwrap();
-        let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() };
-        let destination_ty = destination.ty(&caller_body.local_decls, self.tcx).ty;
-        for arg in args {
-            if !arg.node.ty(&caller_body.local_decls, self.tcx).is_sized(self.tcx, self.typing_env)
-            {
-                // We do not allow inlining functions with unsized params. Inlining these functions
-                // could create unsized locals, which are unsound and being phased out.
-                return Err("Call has unsized argument");
-            }
-        }
-
-        let callee_body = try_instance_mir(self.tcx, callsite.callee.def)?;
-        self.check_mir_body(callsite, callee_body, callee_attrs, cross_crate_inlinable)?;
-
-        let Ok(callee_body) = callsite.callee.try_instantiate_mir_and_normalize_erasing_regions(
-            self.tcx,
-            self.typing_env,
-            ty::EarlyBinder::bind(callee_body.clone()),
-        ) else {
-            return Err("failed to normalize callee body");
-        };
-
-        // Normally, this shouldn't be required, but trait normalization failure can create a
-        // validation ICE.
-        if !validate_types(self.tcx, self.typing_env, &callee_body, &caller_body).is_empty() {
-            return Err("failed to validate callee body");
-        }
-
-        // Check call signature compatibility.
-        // Normally, this shouldn't be required, but trait normalization failure can create a
-        // validation ICE.
-        let output_type = callee_body.return_ty();
-        if !util::sub_types(self.tcx, self.typing_env, output_type, destination_ty) {
-            trace!(?output_type, ?destination_ty);
-            return Err("failed to normalize return type");
-        }
-        if callsite.fn_sig.abi() == ExternAbi::RustCall {
-            // FIXME: Don't inline user-written `extern "rust-call"` functions,
-            // since this is generally perf-negative on rustc, and we hope that
-            // LLVM will inline these functions instead.
-            if callee_body.spread_arg.is_some() {
-                return Err("do not inline user-written rust-call functions");
-            }
+    fn tcx(&self) -> TyCtxt<'tcx> {
+        self.tcx
+    }
 
-            let (self_arg, arg_tuple) = match &args[..] {
-                [arg_tuple] => (None, arg_tuple),
-                [self_arg, arg_tuple] => (Some(self_arg), arg_tuple),
-                _ => bug!("Expected `rust-call` to have 1 or 2 args"),
-            };
+    fn typing_env(&self) -> ty::TypingEnv<'tcx> {
+        self.typing_env
+    }
 
-            let self_arg_ty =
-                self_arg.map(|self_arg| self_arg.node.ty(&caller_body.local_decls, self.tcx));
+    fn history(&self) -> &[DefId] {
+        &self.history
+    }
 
-            let arg_tuple_ty = arg_tuple.node.ty(&caller_body.local_decls, self.tcx);
-            let ty::Tuple(arg_tuple_tys) = *arg_tuple_ty.kind() else {
-                bug!("Closure arguments are not passed as a tuple");
-            };
+    fn caller_def_id(&self) -> DefId {
+        self.def_id
+    }
 
-            for (arg_ty, input) in
-                self_arg_ty.into_iter().chain(arg_tuple_tys).zip(callee_body.args_iter())
-            {
-                let input_type = callee_body.local_decls[input].ty;
-                if !util::sub_types(self.tcx, self.typing_env, input_type, arg_ty) {
-                    trace!(?arg_ty, ?input_type);
-                    return Err("failed to normalize tuple argument type");
-                }
-            }
-        } else {
-            for (arg, input) in args.iter().zip(callee_body.args_iter()) {
-                let input_type = callee_body.local_decls[input].ty;
-                let arg_ty = arg.node.ty(&caller_body.local_decls, self.tcx);
-                if !util::sub_types(self.tcx, self.typing_env, input_type, arg_ty) {
-                    trace!(?arg_ty, ?input_type);
-                    return Err("failed to normalize argument type");
-                }
-            }
-        }
+    fn changed(self) -> bool {
+        self.changed
+    }
 
-        let old_blocks = caller_body.basic_blocks.next_index();
-        self.inline_call(caller_body, callsite, callee_body);
-        let new_blocks = old_blocks..caller_body.basic_blocks.next_index();
+    fn should_inline_for_callee(&self, def_id: DefId) -> bool {
+        ForceInline::should_run_pass_for_callee(self.tcx(), def_id)
+    }
 
-        Ok(new_blocks)
+    fn check_caller_mir_body(&self, _: &Body<'tcx>) -> bool {
+        true
     }
 
-    fn check_mir_is_available(
+    #[instrument(level = "debug", skip(self, callee_body))]
+    fn check_callee_mir_body(
         &self,
-        caller_body: &Body<'tcx>,
-        callee: Instance<'tcx>,
+        _: &CallSite<'tcx>,
+        callee_body: &Body<'tcx>,
+        callee_attrs: &CodegenFnAttrs,
     ) -> Result<(), &'static str> {
-        let caller_def_id = caller_body.source.def_id();
-        let callee_def_id = callee.def_id();
-        if callee_def_id == caller_def_id {
-            return Err("self-recursion");
-        }
-
-        match callee.def {
-            InstanceKind::Item(_) => {
-                // If there is no MIR available (either because it was not in metadata or
-                // because it has no MIR because it's an extern function), then the inliner
-                // won't cause cycles on this.
-                if !self.tcx.is_mir_available(callee_def_id) {
-                    return Err("item MIR unavailable");
-                }
-            }
-            // These have no own callable MIR.
-            InstanceKind::Intrinsic(_) | InstanceKind::Virtual(..) => {
-                return Err("instance without MIR (intrinsic / virtual)");
-            }
-
-            // FIXME(#127030): `ConstParamHasTy` has bad interactions with
-            // the drop shim builder, which does not evaluate predicates in
-            // the correct param-env for types being dropped. Stall resolving
-            // the MIR for this instance until all of its const params are
-            // substituted.
-            InstanceKind::DropGlue(_, Some(ty)) if ty.has_type_flags(TypeFlags::HAS_CT_PARAM) => {
-                return Err("still needs substitution");
-            }
-
-            // This cannot result in an immediate cycle since the callee MIR is a shim, which does
-            // not get any optimizations run on it. Any subsequent inlining may cause cycles, but we
-            // do not need to catch this here, we can wait until the inliner decides to continue
-            // inlining a second time.
-            InstanceKind::VTableShim(_)
-            | InstanceKind::ReifyShim(..)
-            | InstanceKind::FnPtrShim(..)
-            | InstanceKind::ClosureOnceShim { .. }
-            | InstanceKind::ConstructCoroutineInClosureShim { .. }
-            | InstanceKind::DropGlue(..)
-            | InstanceKind::CloneShim(..)
-            | InstanceKind::ThreadLocalShim(..)
-            | InstanceKind::FnPtrAddrShim(..)
-            | InstanceKind::AsyncDropGlueCtorShim(..) => return Ok(()),
+        if callee_body.tainted_by_errors.is_some() {
+            return Err("body has errors");
         }
 
-        if self.tcx.is_constructor(callee_def_id) {
-            trace!("constructors always have MIR");
-            // Constructor functions cannot cause a query cycle.
-            return Ok(());
-        }
-
-        if callee_def_id.is_local() {
-            // If we know for sure that the function we're calling will itself try to
-            // call us, then we avoid inlining that function.
-            if self.tcx.mir_callgraph_reachable((callee, caller_def_id.expect_local())) {
-                return Err("caller might be reachable from callee (query cycle avoidance)");
-            }
-
-            Ok(())
+        let caller_attrs = self.tcx().codegen_fn_attrs(self.caller_def_id());
+        if callee_attrs.instruction_set != caller_attrs.instruction_set
+            && callee_body
+                .basic_blocks
+                .iter()
+                .any(|bb| matches!(bb.terminator().kind, TerminatorKind::InlineAsm { .. }))
+        {
+            // During the attribute checking stage we allow a callee with no
+            // instruction_set assigned to count as compatible with a function that does
+            // assign one. However, during this stage we require an exact match when any
+            // inline-asm is detected. LLVM will still possibly do an inline later on
+            // if the no-attribute function ends up with the same instruction set anyway.
+            Err("cannot move inline-asm across instruction sets")
         } else {
-            // This cannot result in an immediate cycle since the callee MIR is from another crate
-            // and is already optimized. Any subsequent inlining may cause cycles, but we do
-            // not need to catch this here, we can wait until the inliner decides to continue
-            // inlining a second time.
-            trace!("functions from other crates always have MIR");
             Ok(())
         }
     }
 
-    fn resolve_callsite(
-        &self,
-        caller_body: &Body<'tcx>,
-        bb: BasicBlock,
-        bb_data: &BasicBlockData<'tcx>,
-    ) -> Option<CallSite<'tcx>> {
-        // Only consider direct calls to functions
-        let terminator = bb_data.terminator();
-
-        // FIXME(explicit_tail_calls): figure out if we can inline tail calls
-        if let TerminatorKind::Call { ref func, fn_span, .. } = terminator.kind {
-            let func_ty = func.ty(caller_body, self.tcx);
-            if let ty::FnDef(def_id, args) = *func_ty.kind() {
-                // To resolve an instance its args have to be fully normalized.
-                let args = self.tcx.try_normalize_erasing_regions(self.typing_env, args).ok()?;
-                let callee = Instance::try_resolve(self.tcx, self.typing_env, def_id, args)
-                    .ok()
-                    .flatten()?;
-
-                if let InstanceKind::Virtual(..) | InstanceKind::Intrinsic(_) = callee.def {
-                    return None;
-                }
-
-                if self.history.contains(&callee.def_id()) {
-                    return None;
-                }
+    fn inline_limit_for_block(&self) -> Option<usize> {
+        Some(usize::MAX)
+    }
 
-                let fn_sig = self.tcx.fn_sig(def_id).instantiate(self.tcx, args);
+    fn on_inline_success(
+        &mut self,
+        callsite: &CallSite<'tcx>,
+        caller_body: &mut Body<'tcx>,
+        new_blocks: std::ops::Range<BasicBlock>,
+    ) {
+        self.changed = true;
 
-                // Additionally, check that the body that we're inlining actually agrees
-                // with the ABI of the trait that the item comes from.
-                if let InstanceKind::Item(instance_def_id) = callee.def
-                    && self.tcx.def_kind(instance_def_id) == DefKind::AssocFn
-                    && let instance_fn_sig = self.tcx.fn_sig(instance_def_id).skip_binder()
-                    && instance_fn_sig.abi() != fn_sig.abi()
-                {
-                    return None;
-                }
+        self.history.push(callsite.callee.def_id());
+        process_blocks(self, caller_body, new_blocks);
+        self.history.pop();
+    }
 
-                let source_info = SourceInfo { span: fn_span, ..terminator.source_info };
+    fn on_inline_failure(&self, callsite: &CallSite<'tcx>, reason: &'static str) {
+        let tcx = self.tcx();
+        let InlineAttr::Force { attr_span, reason: justification } =
+            tcx.codegen_fn_attrs(callsite.callee.def_id()).inline
+        else {
+            bug!("called on item without required inlining");
+        };
 
-                return Some(CallSite { callee, fn_sig, block: bb, source_info });
-            }
-        }
+        let call_span = callsite.source_info.span;
+        tcx.dcx().emit_err(crate::errors::ForceInlineFailure {
+            call_span,
+            attr_span,
+            caller_span: tcx.def_span(self.def_id),
+            caller: tcx.def_path_str(self.def_id),
+            callee_span: tcx.def_span(callsite.callee.def_id()),
+            callee: tcx.def_path_str(callsite.callee.def_id()),
+            reason,
+            justification: justification.map(|sym| crate::errors::ForceInlineJustification { sym }),
+        });
+    }
 
-        None
+    fn on_inline_limit_reached(&self) -> bool {
+        false
     }
+}
 
-    /// Returns an error if inlining is not possible based on codegen attributes alone. A success
-    /// indicates that inlining decision should be based on other criteria.
-    fn check_codegen_attributes(
-        &self,
-        callsite: &CallSite<'tcx>,
-        callee_attrs: &CodegenFnAttrs,
-        cross_crate_inlinable: bool,
-    ) -> Result<(), &'static str> {
-        if self.tcx.has_attr(callsite.callee.def_id(), sym::rustc_no_mir_inline) {
-            return Err("#[rustc_no_mir_inline]");
-        }
+struct NormalInliner<'tcx> {
+    tcx: TyCtxt<'tcx>,
+    typing_env: ty::TypingEnv<'tcx>,
+    /// `DefId` of caller.
+    def_id: DefId,
+    /// Stack of inlined instances.
+    /// We only check the `DefId` and not the args because we want to
+    /// avoid inlining cases of polymorphic recursion.
+    /// The number of `DefId`s is finite, so checking history is enough
+    /// to ensure that we do not loop endlessly while inlining.
+    history: Vec<DefId>,
+    /// Indicates that the caller body has been modified.
+    changed: bool,
+    /// Indicates that the caller is #[inline] and just calls another function,
+    /// and thus we can inline less into it as it'll be inlined itself.
+    caller_is_inline_forwarder: bool,
+}
 
-        if let InlineAttr::Never = callee_attrs.inline {
-            return Err("never inline hint");
+impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> {
+    fn new(tcx: TyCtxt<'tcx>, def_id: DefId, body: &Body<'tcx>) -> Self {
+        let typing_env = body.typing_env(tcx);
+        let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id);
+
+        Self {
+            tcx,
+            typing_env,
+            def_id,
+            history: Vec::new(),
+            changed: false,
+            caller_is_inline_forwarder: matches!(
+                codegen_fn_attrs.inline,
+                InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. }
+            ) && body_is_forwarder(body),
         }
+    }
 
-        // Reachability pass defines which functions are eligible for inlining. Generally inlining
-        // other functions is incorrect because they could reference symbols that aren't exported.
-        let is_generic = callsite.callee.args.non_erasable_generics().next().is_some();
-        if !is_generic && !cross_crate_inlinable {
-            return Err("not exported");
-        }
+    fn tcx(&self) -> TyCtxt<'tcx> {
+        self.tcx
+    }
 
-        if callsite.fn_sig.c_variadic() {
-            return Err("C variadic");
-        }
+    fn caller_def_id(&self) -> DefId {
+        self.def_id
+    }
 
-        if callee_attrs.flags.contains(CodegenFnAttrFlags::COLD) {
-            return Err("cold");
-        }
+    fn typing_env(&self) -> ty::TypingEnv<'tcx> {
+        self.typing_env
+    }
 
-        if callee_attrs.no_sanitize != self.codegen_fn_attrs.no_sanitize {
-            return Err("incompatible sanitizer set");
-        }
+    fn history(&self) -> &[DefId] {
+        &self.history
+    }
 
-        // Two functions are compatible if the callee has no attribute (meaning
-        // that it's codegen agnostic), or sets an attribute that is identical
-        // to this function's attribute.
-        if callee_attrs.instruction_set.is_some()
-            && callee_attrs.instruction_set != self.codegen_fn_attrs.instruction_set
-        {
-            return Err("incompatible instruction set");
-        }
+    fn changed(self) -> bool {
+        self.changed
+    }
+
+    fn should_inline_for_callee(&self, _: DefId) -> bool {
+        true
+    }
 
-        let callee_feature_names = callee_attrs.target_features.iter().map(|f| f.name);
-        let this_feature_names = self.codegen_fn_attrs.target_features.iter().map(|f| f.name);
-        if callee_feature_names.ne(this_feature_names) {
-            // In general it is not correct to inline a callee with target features that are a
-            // subset of the caller. This is because the callee might contain calls, and the ABI of
-            // those calls depends on the target features of the surrounding function. By moving a
-            // `Call` terminator from one MIR body to another with more target features, we might
-            // change the ABI of that call!
-            return Err("incompatible target features");
+    fn check_caller_mir_body(&self, body: &Body<'tcx>) -> bool {
+        // Avoid inlining into coroutines, since their `optimized_mir` is used for layout computation,
+        // which can create a cycle, even when no attempt is made to inline the function in the other
+        // direction.
+        if body.coroutine.is_some() {
+            return false;
         }
 
-        Ok(())
+        true
     }
 
-    /// Returns inlining decision that is based on the examination of callee MIR body.
-    /// Assumes that codegen attributes have been checked for compatibility already.
     #[instrument(level = "debug", skip(self, callee_body))]
-    fn check_mir_body(
+    fn check_callee_mir_body(
         &self,
         callsite: &CallSite<'tcx>,
         callee_body: &Body<'tcx>,
         callee_attrs: &CodegenFnAttrs,
-        cross_crate_inlinable: bool,
     ) -> Result<(), &'static str> {
-        let tcx = self.tcx;
+        let tcx = self.tcx();
 
         if let Some(_) = callee_body.tainted_by_errors {
-            return Err("Body is tainted");
+            return Err("body has errors");
         }
 
         let mut threshold = if self.caller_is_inline_forwarder {
-            self.tcx.sess.opts.unstable_opts.inline_mir_forwarder_threshold.unwrap_or(30)
-        } else if cross_crate_inlinable {
-            self.tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100)
+            tcx.sess.opts.unstable_opts.inline_mir_forwarder_threshold.unwrap_or(30)
+        } else if tcx.cross_crate_inlinable(callsite.callee.def_id()) {
+            tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100)
         } else {
-            self.tcx.sess.opts.unstable_opts.inline_mir_threshold.unwrap_or(50)
+            tcx.sess.opts.unstable_opts.inline_mir_threshold.unwrap_or(50)
         };
 
         // Give a bonus functions with a small number of blocks,
@@ -497,13 +363,13 @@ impl<'tcx> Inliner<'tcx> {
         // FIXME: Give a bonus to functions with only a single caller
 
         let mut checker =
-            CostChecker::new(self.tcx, self.typing_env, Some(callsite.callee), callee_body);
+            CostChecker::new(tcx, self.typing_env(), Some(callsite.callee), callee_body);
 
         checker.add_function_level_costs();
 
         // Traverse the MIR manually so we can account for the effects of inlining on the CFG.
         let mut work_list = vec![START_BLOCK];
-        let mut visited = BitSet::new_empty(callee_body.basic_blocks.len());
+        let mut visited = DenseBitSet::new_empty(callee_body.basic_blocks.len());
         while let Some(bb) = work_list.pop() {
             if !visited.insert(bb.index()) {
                 continue;
@@ -513,20 +379,20 @@ impl<'tcx> Inliner<'tcx> {
             checker.visit_basic_block_data(bb, blk);
 
             let term = blk.terminator();
+            let caller_attrs = tcx.codegen_fn_attrs(self.caller_def_id());
             if let TerminatorKind::Drop { ref place, target, unwind, replace: _ } = term.kind {
                 work_list.push(target);
 
                 // If the place doesn't actually need dropping, treat it like a regular goto.
-                let ty = callsite.callee.instantiate_mir(
-                    self.tcx,
-                    ty::EarlyBinder::bind(&place.ty(callee_body, tcx).ty),
-                );
-                if ty.needs_drop(tcx, self.typing_env)
+                let ty = callsite
+                    .callee
+                    .instantiate_mir(tcx, ty::EarlyBinder::bind(&place.ty(callee_body, tcx).ty));
+                if ty.needs_drop(tcx, self.typing_env())
                     && let UnwindAction::Cleanup(unwind) = unwind
                 {
                     work_list.push(unwind);
                 }
-            } else if callee_attrs.instruction_set != self.codegen_fn_attrs.instruction_set
+            } else if callee_attrs.instruction_set != caller_attrs.instruction_set
                 && matches!(term.kind, TerminatorKind::InlineAsm { .. })
             {
                 // During the attribute checking stage we allow a callee with no
@@ -534,7 +400,7 @@ impl<'tcx> Inliner<'tcx> {
                 // assign one. However, during this stage we require an exact match when any
                 // inline-asm is detected. LLVM will still possibly do an inline later on
                 // if the no-attribute function ends up with the same instruction set anyway.
-                return Err("Cannot move inline-asm across instruction sets");
+                return Err("cannot move inline-asm across instruction sets");
             } else if let TerminatorKind::TailCall { .. } = term.kind {
                 // FIXME(explicit_tail_calls): figure out how exactly functions containing tail
                 // calls can be inlined (and if they even should)
@@ -558,321 +424,688 @@ impl<'tcx> Inliner<'tcx> {
         }
     }
 
-    fn inline_call(
-        &self,
-        caller_body: &mut Body<'tcx>,
+    fn inline_limit_for_block(&self) -> Option<usize> {
+        match self.history.len() {
+            0 => Some(usize::MAX),
+            1..=TOP_DOWN_DEPTH_LIMIT => Some(1),
+            _ => None,
+        }
+    }
+
+    fn on_inline_success(
+        &mut self,
         callsite: &CallSite<'tcx>,
-        mut callee_body: Body<'tcx>,
+        caller_body: &mut Body<'tcx>,
+        new_blocks: std::ops::Range<BasicBlock>,
     ) {
-        let terminator = caller_body[callsite.block].terminator.take().unwrap();
-        let TerminatorKind::Call { func, args, destination, unwind, target, .. } = terminator.kind
-        else {
-            bug!("unexpected terminator kind {:?}", terminator.kind);
-        };
+        self.changed = true;
 
-        let return_block = if let Some(block) = target {
-            // Prepare a new block for code that should execute when call returns. We don't use
-            // target block directly since it might have other predecessors.
-            let data = BasicBlockData::new(
-                Some(Terminator {
-                    source_info: terminator.source_info,
-                    kind: TerminatorKind::Goto { target: block },
-                }),
-                caller_body[block].is_cleanup,
-            );
-            Some(caller_body.basic_blocks_mut().push(data))
-        } else {
-            None
+        self.history.push(callsite.callee.def_id());
+        process_blocks(self, caller_body, new_blocks);
+        self.history.pop();
+    }
+
+    fn on_inline_limit_reached(&self) -> bool {
+        true
+    }
+
+    fn on_inline_failure(&self, _: &CallSite<'tcx>, _: &'static str) {}
+}
+
+fn inline<'tcx, T: Inliner<'tcx>>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
+    let def_id = body.source.def_id();
+
+    // Only do inlining into fn bodies.
+    if !tcx.hir().body_owner_kind(def_id).is_fn_or_closure() {
+        return false;
+    }
+
+    let mut inliner = T::new(tcx, def_id, body);
+    if !inliner.check_caller_mir_body(body) {
+        return false;
+    }
+
+    let blocks = START_BLOCK..body.basic_blocks.next_index();
+    process_blocks(&mut inliner, body, blocks);
+    inliner.changed()
+}
+
+fn process_blocks<'tcx, I: Inliner<'tcx>>(
+    inliner: &mut I,
+    caller_body: &mut Body<'tcx>,
+    blocks: Range<BasicBlock>,
+) {
+    let Some(inline_limit) = inliner.inline_limit_for_block() else { return };
+    let mut inlined_count = 0;
+    for bb in blocks {
+        let bb_data = &caller_body[bb];
+        if bb_data.is_cleanup {
+            continue;
+        }
+
+        let Some(callsite) = resolve_callsite(inliner, caller_body, bb, bb_data) else {
+            continue;
         };
 
-        // If the call is something like `a[*i] = f(i)`, where
-        // `i : &mut usize`, then just duplicating the `a[*i]`
-        // Place could result in two different locations if `f`
-        // writes to `i`. To prevent this we need to create a temporary
-        // borrow of the place and pass the destination as `*temp` instead.
-        fn dest_needs_borrow(place: Place<'_>) -> bool {
-            for elem in place.projection.iter() {
-                match elem {
-                    ProjectionElem::Deref | ProjectionElem::Index(_) => return true,
-                    _ => {}
+        let span = trace_span!("process_blocks", %callsite.callee, ?bb);
+        let _guard = span.enter();
+
+        match try_inlining(inliner, caller_body, &callsite) {
+            Err(reason) => {
+                debug!("not-inlined {} [{}]", callsite.callee, reason);
+                inliner.on_inline_failure(&callsite, reason);
+            }
+            Ok(new_blocks) => {
+                debug!("inlined {}", callsite.callee);
+                inliner.on_inline_success(&callsite, caller_body, new_blocks);
+
+                inlined_count += 1;
+                if inlined_count == inline_limit {
+                    if inliner.on_inline_limit_reached() {
+                        return;
+                    }
                 }
             }
+        }
+    }
+}
+
+fn resolve_callsite<'tcx, I: Inliner<'tcx>>(
+    inliner: &I,
+    caller_body: &Body<'tcx>,
+    bb: BasicBlock,
+    bb_data: &BasicBlockData<'tcx>,
+) -> Option<CallSite<'tcx>> {
+    let tcx = inliner.tcx();
+    // Only consider direct calls to functions
+    let terminator = bb_data.terminator();
+
+    // FIXME(explicit_tail_calls): figure out if we can inline tail calls
+    if let TerminatorKind::Call { ref func, fn_span, .. } = terminator.kind {
+        let func_ty = func.ty(caller_body, tcx);
+        if let ty::FnDef(def_id, args) = *func_ty.kind() {
+            if !inliner.should_inline_for_callee(def_id) {
+                debug!("not enabled");
+                return None;
+            }
+
+            // To resolve an instance its args have to be fully normalized.
+            let args = tcx.try_normalize_erasing_regions(inliner.typing_env(), args).ok()?;
+            let callee =
+                Instance::try_resolve(tcx, inliner.typing_env(), def_id, args).ok().flatten()?;
+
+            if let InstanceKind::Virtual(..) | InstanceKind::Intrinsic(_) = callee.def {
+                return None;
+            }
+
+            if inliner.history().contains(&callee.def_id()) {
+                return None;
+            }
 
-            false
+            let fn_sig = tcx.fn_sig(def_id).instantiate(tcx, args);
+
+            // Additionally, check that the body that we're inlining actually agrees
+            // with the ABI of the trait that the item comes from.
+            if let InstanceKind::Item(instance_def_id) = callee.def
+                && tcx.def_kind(instance_def_id) == DefKind::AssocFn
+                && let instance_fn_sig = tcx.fn_sig(instance_def_id).skip_binder()
+                && instance_fn_sig.abi() != fn_sig.abi()
+            {
+                return None;
+            }
+
+            let source_info = SourceInfo { span: fn_span, ..terminator.source_info };
+
+            return Some(CallSite { callee, fn_sig, block: bb, source_info });
         }
+    }
 
-        let dest = if dest_needs_borrow(destination) {
-            trace!("creating temp for return destination");
-            let dest = Rvalue::Ref(
-                self.tcx.lifetimes.re_erased,
-                BorrowKind::Mut { kind: MutBorrowKind::Default },
-                destination,
-            );
-            let dest_ty = dest.ty(caller_body, self.tcx);
-            let temp =
-                Place::from(self.new_call_temp(caller_body, callsite, dest_ty, return_block));
-            caller_body[callsite.block].statements.push(Statement {
-                source_info: callsite.source_info,
-                kind: StatementKind::Assign(Box::new((temp, dest))),
-            });
-            self.tcx.mk_place_deref(temp)
-        } else {
-            destination
-        };
+    None
+}
 
-        // Always create a local to hold the destination, as `RETURN_PLACE` may appear
-        // where a full `Place` is not allowed.
-        let (remap_destination, destination_local) = if let Some(d) = dest.as_local() {
-            (false, d)
-        } else {
-            (
-                true,
-                self.new_call_temp(
-                    caller_body,
-                    callsite,
-                    destination.ty(caller_body, self.tcx).ty,
-                    return_block,
-                ),
-            )
-        };
+/// Attempts to inline a callsite into the caller body. When successful returns basic blocks
+/// containing the inlined body. Otherwise returns an error describing why inlining didn't take
+/// place.
+fn try_inlining<'tcx, I: Inliner<'tcx>>(
+    inliner: &I,
+    caller_body: &mut Body<'tcx>,
+    callsite: &CallSite<'tcx>,
+) -> Result<std::ops::Range<BasicBlock>, &'static str> {
+    let tcx = inliner.tcx();
+    check_mir_is_available(inliner, caller_body, callsite.callee)?;
+
+    let callee_attrs = tcx.codegen_fn_attrs(callsite.callee.def_id());
+    rustc_mir_build::check_inline::is_inline_valid_on_fn(tcx, callsite.callee.def_id())?;
+    check_codegen_attributes(inliner, callsite, callee_attrs)?;
+
+    let terminator = caller_body[callsite.block].terminator.as_ref().unwrap();
+    let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() };
+    let destination_ty = destination.ty(&caller_body.local_decls, tcx).ty;
+    for arg in args {
+        if !arg.node.ty(&caller_body.local_decls, tcx).is_sized(tcx, inliner.typing_env()) {
+            // We do not allow inlining functions with unsized params. Inlining these functions
+            // could create unsized locals, which are unsound and being phased out.
+            return Err("call has unsized argument");
+        }
+    }
 
-        // Copy the arguments if needed.
-        let args = self.make_call_args(args, callsite, caller_body, &callee_body, return_block);
+    let callee_body = try_instance_mir(tcx, callsite.callee.def)?;
+    rustc_mir_build::check_inline::is_inline_valid_on_body(tcx, callee_body)?;
+    inliner.check_callee_mir_body(callsite, callee_body, callee_attrs)?;
 
-        let mut integrator = Integrator {
-            args: &args,
-            new_locals: Local::new(caller_body.local_decls.len())..,
-            new_scopes: SourceScope::new(caller_body.source_scopes.len())..,
-            new_blocks: BasicBlock::new(caller_body.basic_blocks.len())..,
-            destination: destination_local,
-            callsite_scope: caller_body.source_scopes[callsite.source_info.scope].clone(),
-            callsite,
-            cleanup_block: unwind,
-            in_cleanup_block: false,
-            return_block,
-            tcx: self.tcx,
-            always_live_locals: BitSet::new_filled(callee_body.local_decls.len()),
+    let Ok(callee_body) = callsite.callee.try_instantiate_mir_and_normalize_erasing_regions(
+        tcx,
+        inliner.typing_env(),
+        ty::EarlyBinder::bind(callee_body.clone()),
+    ) else {
+        debug!("failed to normalize callee body");
+        return Err("implementation limitation");
+    };
+
+    // Normally, this shouldn't be required, but trait normalization failure can create a
+    // validation ICE.
+    if !validate_types(tcx, inliner.typing_env(), &callee_body, &caller_body).is_empty() {
+        debug!("failed to validate callee body");
+        return Err("implementation limitation");
+    }
+
+    // Check call signature compatibility.
+    // Normally, this shouldn't be required, but trait normalization failure can create a
+    // validation ICE.
+    let output_type = callee_body.return_ty();
+    if !util::sub_types(tcx, inliner.typing_env(), output_type, destination_ty) {
+        trace!(?output_type, ?destination_ty);
+        debug!("failed to normalize return type");
+        return Err("implementation limitation");
+    }
+    if callsite.fn_sig.abi() == ExternAbi::RustCall {
+        // FIXME: Don't inline user-written `extern "rust-call"` functions,
+        // since this is generally perf-negative on rustc, and we hope that
+        // LLVM will inline these functions instead.
+        if callee_body.spread_arg.is_some() {
+            return Err("user-written rust-call functions");
+        }
+
+        let (self_arg, arg_tuple) = match &args[..] {
+            [arg_tuple] => (None, arg_tuple),
+            [self_arg, arg_tuple] => (Some(self_arg), arg_tuple),
+            _ => bug!("Expected `rust-call` to have 1 or 2 args"),
         };
 
-        // Map all `Local`s, `SourceScope`s and `BasicBlock`s to new ones
-        // (or existing ones, in a few special cases) in the caller.
-        integrator.visit_body(&mut callee_body);
+        let self_arg_ty = self_arg.map(|self_arg| self_arg.node.ty(&caller_body.local_decls, tcx));
 
-        // If there are any locals without storage markers, give them storage only for the
-        // duration of the call.
-        for local in callee_body.vars_and_temps_iter() {
-            if integrator.always_live_locals.contains(local) {
-                let new_local = integrator.map_local(local);
-                caller_body[callsite.block].statements.push(Statement {
-                    source_info: callsite.source_info,
-                    kind: StatementKind::StorageLive(new_local),
-                });
+        let arg_tuple_ty = arg_tuple.node.ty(&caller_body.local_decls, tcx);
+        let ty::Tuple(arg_tuple_tys) = *arg_tuple_ty.kind() else {
+            bug!("Closure arguments are not passed as a tuple");
+        };
+
+        for (arg_ty, input) in
+            self_arg_ty.into_iter().chain(arg_tuple_tys).zip(callee_body.args_iter())
+        {
+            let input_type = callee_body.local_decls[input].ty;
+            if !util::sub_types(tcx, inliner.typing_env(), input_type, arg_ty) {
+                trace!(?arg_ty, ?input_type);
+                debug!("failed to normalize tuple argument type");
+                return Err("implementation limitation");
             }
         }
-        if let Some(block) = return_block {
-            // To avoid repeated O(n) insert, push any new statements to the end and rotate
-            // the slice once.
-            let mut n = 0;
-            if remap_destination {
-                caller_body[block].statements.push(Statement {
-                    source_info: callsite.source_info,
-                    kind: StatementKind::Assign(Box::new((
-                        dest,
-                        Rvalue::Use(Operand::Move(destination_local.into())),
-                    ))),
-                });
-                n += 1;
+    } else {
+        for (arg, input) in args.iter().zip(callee_body.args_iter()) {
+            let input_type = callee_body.local_decls[input].ty;
+            let arg_ty = arg.node.ty(&caller_body.local_decls, tcx);
+            if !util::sub_types(tcx, inliner.typing_env(), input_type, arg_ty) {
+                trace!(?arg_ty, ?input_type);
+                debug!("failed to normalize argument type");
+                return Err("implementation limitation");
             }
-            for local in callee_body.vars_and_temps_iter().rev() {
-                if integrator.always_live_locals.contains(local) {
-                    let new_local = integrator.map_local(local);
-                    caller_body[block].statements.push(Statement {
-                        source_info: callsite.source_info,
-                        kind: StatementKind::StorageDead(new_local),
-                    });
-                    n += 1;
-                }
+        }
+    }
+
+    let old_blocks = caller_body.basic_blocks.next_index();
+    inline_call(inliner, caller_body, callsite, callee_body);
+    let new_blocks = old_blocks..caller_body.basic_blocks.next_index();
+
+    Ok(new_blocks)
+}
+
+fn check_mir_is_available<'tcx, I: Inliner<'tcx>>(
+    inliner: &I,
+    caller_body: &Body<'tcx>,
+    callee: Instance<'tcx>,
+) -> Result<(), &'static str> {
+    let caller_def_id = caller_body.source.def_id();
+    let callee_def_id = callee.def_id();
+    if callee_def_id == caller_def_id {
+        return Err("self-recursion");
+    }
+
+    match callee.def {
+        InstanceKind::Item(_) => {
+            // If there is no MIR available (either because it was not in metadata or
+            // because it has no MIR because it's an extern function), then the inliner
+            // won't cause cycles on this.
+            if !inliner.tcx().is_mir_available(callee_def_id) {
+                debug!("item MIR unavailable");
+                return Err("implementation limitation");
             }
-            caller_body[block].statements.rotate_right(n);
+        }
+        // These have no own callable MIR.
+        InstanceKind::Intrinsic(_) | InstanceKind::Virtual(..) => {
+            debug!("instance without MIR (intrinsic / virtual)");
+            return Err("implementation limitation");
         }
 
-        // Insert all of the (mapped) parts of the callee body into the caller.
-        caller_body.local_decls.extend(callee_body.drain_vars_and_temps());
-        caller_body.source_scopes.append(&mut callee_body.source_scopes);
-        if self
-            .tcx
-            .sess
-            .opts
-            .unstable_opts
-            .inline_mir_preserve_debug
-            .unwrap_or(self.tcx.sess.opts.debuginfo != DebugInfo::None)
-        {
-            // Note that we need to preserve these in the standard library so that
-            // people working on rust can build with or without debuginfo while
-            // still getting consistent results from the mir-opt tests.
-            caller_body.var_debug_info.append(&mut callee_body.var_debug_info);
+        // FIXME(#127030): `ConstParamHasTy` has bad interactions with
+        // the drop shim builder, which does not evaluate predicates in
+        // the correct param-env for types being dropped. Stall resolving
+        // the MIR for this instance until all of its const params are
+        // substituted.
+        InstanceKind::DropGlue(_, Some(ty)) if ty.has_type_flags(TypeFlags::HAS_CT_PARAM) => {
+            debug!("still needs substitution");
+            return Err("implementation limitation");
         }
-        caller_body.basic_blocks_mut().append(callee_body.basic_blocks_mut());
 
-        caller_body[callsite.block].terminator = Some(Terminator {
-            source_info: callsite.source_info,
-            kind: TerminatorKind::Goto { target: integrator.map_block(START_BLOCK) },
-        });
+        // This cannot result in an immediate cycle since the callee MIR is a shim, which does
+        // not get any optimizations run on it. Any subsequent inlining may cause cycles, but we
+        // do not need to catch this here, we can wait until the inliner decides to continue
+        // inlining a second time.
+        InstanceKind::VTableShim(_)
+        | InstanceKind::ReifyShim(..)
+        | InstanceKind::FnPtrShim(..)
+        | InstanceKind::ClosureOnceShim { .. }
+        | InstanceKind::ConstructCoroutineInClosureShim { .. }
+        | InstanceKind::DropGlue(..)
+        | InstanceKind::CloneShim(..)
+        | InstanceKind::ThreadLocalShim(..)
+        | InstanceKind::FnPtrAddrShim(..)
+        | InstanceKind::AsyncDropGlueCtorShim(..) => return Ok(()),
+    }
 
-        // Copy required constants from the callee_body into the caller_body. Although we are only
-        // pushing unevaluated consts to `required_consts`, here they may have been evaluated
-        // because we are calling `instantiate_and_normalize_erasing_regions` -- so we filter again.
-        caller_body.required_consts.as_mut().unwrap().extend(
-            callee_body.required_consts().into_iter().filter(|ct| ct.const_.is_required_const()),
-        );
-        // Now that we incorporated the callee's `required_consts`, we can remove the callee from
-        // `mentioned_items` -- but we have to take their `mentioned_items` in return. This does
-        // some extra work here to save the monomorphization collector work later. It helps a lot,
-        // since monomorphization can avoid a lot of work when the "mentioned items" are similar to
-        // the actually used items. By doing this we can entirely avoid visiting the callee!
-        // We need to reconstruct the `required_item` for the callee so that we can find and
-        // remove it.
-        let callee_item = MentionedItem::Fn(func.ty(caller_body, self.tcx));
-        let caller_mentioned_items = caller_body.mentioned_items.as_mut().unwrap();
-        if let Some(idx) = caller_mentioned_items.iter().position(|item| item.node == callee_item) {
-            // We found the callee, so remove it and add its items instead.
-            caller_mentioned_items.remove(idx);
-            caller_mentioned_items.extend(callee_body.mentioned_items());
-        } else {
-            // If we can't find the callee, there's no point in adding its items. Probably it
-            // already got removed by being inlined elsewhere in the same function, so we already
-            // took its items.
+    if inliner.tcx().is_constructor(callee_def_id) {
+        trace!("constructors always have MIR");
+        // Constructor functions cannot cause a query cycle.
+        return Ok(());
+    }
+
+    if callee_def_id.is_local()
+        && !inliner
+            .tcx()
+            .is_lang_item(inliner.tcx().parent(caller_def_id), rustc_hir::LangItem::FnOnce)
+    {
+        // If we know for sure that the function we're calling will itself try to
+        // call us, then we avoid inlining that function.
+        if inliner.tcx().mir_callgraph_reachable((callee, caller_def_id.expect_local())) {
+            debug!("query cycle avoidance");
+            return Err("caller might be reachable from callee");
         }
+
+        Ok(())
+    } else {
+        // This cannot result in an immediate cycle since the callee MIR is from another crate
+        // and is already optimized. Any subsequent inlining may cause cycles, but we do
+        // not need to catch this here, we can wait until the inliner decides to continue
+        // inlining a second time.
+        trace!("functions from other crates always have MIR");
+        Ok(())
     }
+}
 
-    fn make_call_args(
-        &self,
-        args: Box<[Spanned<Operand<'tcx>>]>,
-        callsite: &CallSite<'tcx>,
-        caller_body: &mut Body<'tcx>,
-        callee_body: &Body<'tcx>,
-        return_block: Option<BasicBlock>,
-    ) -> Box<[Local]> {
-        let tcx = self.tcx;
-
-        // There is a bit of a mismatch between the *caller* of a closure and the *callee*.
-        // The caller provides the arguments wrapped up in a tuple:
-        //
-        //     tuple_tmp = (a, b, c)
-        //     Fn::call(closure_ref, tuple_tmp)
-        //
-        // meanwhile the closure body expects the arguments (here, `a`, `b`, and `c`)
-        // as distinct arguments. (This is the "rust-call" ABI hack.) Normally, codegen has
-        // the job of unpacking this tuple. But here, we are codegen. =) So we want to create
-        // a vector like
-        //
-        //     [closure_ref, tuple_tmp.0, tuple_tmp.1, tuple_tmp.2]
-        //
-        // Except for one tiny wrinkle: we don't actually want `tuple_tmp.0`. It's more convenient
-        // if we "spill" that into *another* temporary, so that we can map the argument
-        // variable in the callee MIR directly to an argument variable on our side.
-        // So we introduce temporaries like:
-        //
-        //     tmp0 = tuple_tmp.0
-        //     tmp1 = tuple_tmp.1
-        //     tmp2 = tuple_tmp.2
-        //
-        // and the vector is `[closure_ref, tmp0, tmp1, tmp2]`.
-        if callsite.fn_sig.abi() == ExternAbi::RustCall && callee_body.spread_arg.is_none() {
-            // FIXME(edition_2024): switch back to a normal method call.
-            let mut args = <_>::into_iter(args);
-            let self_ = self.create_temp_if_necessary(
-                args.next().unwrap().node,
-                callsite,
-                caller_body,
-                return_block,
-            );
-            let tuple = self.create_temp_if_necessary(
-                args.next().unwrap().node,
-                callsite,
-                caller_body,
-                return_block,
-            );
-            assert!(args.next().is_none());
-
-            let tuple = Place::from(tuple);
-            let ty::Tuple(tuple_tys) = tuple.ty(caller_body, tcx).ty.kind() else {
-                bug!("Closure arguments are not passed as a tuple");
-            };
+/// Returns an error if inlining is not possible based on codegen attributes alone. A success
+/// indicates that inlining decision should be based on other criteria.
+fn check_codegen_attributes<'tcx, I: Inliner<'tcx>>(
+    inliner: &I,
+    callsite: &CallSite<'tcx>,
+    callee_attrs: &CodegenFnAttrs,
+) -> Result<(), &'static str> {
+    let tcx = inliner.tcx();
+    if let InlineAttr::Never = callee_attrs.inline {
+        return Err("never inline attribute");
+    }
 
-            // The `closure_ref` in our example above.
-            let closure_ref_arg = iter::once(self_);
+    // Reachability pass defines which functions are eligible for inlining. Generally inlining
+    // other functions is incorrect because they could reference symbols that aren't exported.
+    let is_generic = callsite.callee.args.non_erasable_generics().next().is_some();
+    if !is_generic && !tcx.cross_crate_inlinable(callsite.callee.def_id()) {
+        return Err("not exported");
+    }
 
-            // The `tmp0`, `tmp1`, and `tmp2` in our example above.
-            let tuple_tmp_args = tuple_tys.iter().enumerate().map(|(i, ty)| {
-                // This is e.g., `tuple_tmp.0` in our example above.
-                let tuple_field = Operand::Move(tcx.mk_place_field(tuple, FieldIdx::new(i), ty));
+    let codegen_fn_attrs = tcx.codegen_fn_attrs(inliner.caller_def_id());
+    if callee_attrs.no_sanitize != codegen_fn_attrs.no_sanitize {
+        return Err("incompatible sanitizer set");
+    }
 
-                // Spill to a local to make e.g., `tmp0`.
-                self.create_temp_if_necessary(tuple_field, callsite, caller_body, return_block)
-            });
+    // Two functions are compatible if the callee has no attribute (meaning
+    // that it's codegen agnostic), or sets an attribute that is identical
+    // to this function's attribute.
+    if callee_attrs.instruction_set.is_some()
+        && callee_attrs.instruction_set != codegen_fn_attrs.instruction_set
+    {
+        return Err("incompatible instruction set");
+    }
 
-            closure_ref_arg.chain(tuple_tmp_args).collect()
-        } else {
-            // FIXME(edition_2024): switch back to a normal method call.
-            <_>::into_iter(args)
-                .map(|a| self.create_temp_if_necessary(a.node, callsite, caller_body, return_block))
-                .collect()
-        }
+    let callee_feature_names = callee_attrs.target_features.iter().map(|f| f.name);
+    let this_feature_names = codegen_fn_attrs.target_features.iter().map(|f| f.name);
+    if callee_feature_names.ne(this_feature_names) {
+        // In general it is not correct to inline a callee with target features that are a
+        // subset of the caller. This is because the callee might contain calls, and the ABI of
+        // those calls depends on the target features of the surrounding function. By moving a
+        // `Call` terminator from one MIR body to another with more target features, we might
+        // change the ABI of that call!
+        return Err("incompatible target features");
     }
 
-    /// If `arg` is already a temporary, returns it. Otherwise, introduces a fresh
-    /// temporary `T` and an instruction `T = arg`, and returns `T`.
-    fn create_temp_if_necessary(
-        &self,
-        arg: Operand<'tcx>,
-        callsite: &CallSite<'tcx>,
-        caller_body: &mut Body<'tcx>,
-        return_block: Option<BasicBlock>,
-    ) -> Local {
-        // Reuse the operand if it is a moved temporary.
-        if let Operand::Move(place) = &arg
-            && let Some(local) = place.as_local()
-            && caller_body.local_kind(local) == LocalKind::Temp
-        {
-            return local;
+    Ok(())
+}
+
+fn inline_call<'tcx, I: Inliner<'tcx>>(
+    inliner: &I,
+    caller_body: &mut Body<'tcx>,
+    callsite: &CallSite<'tcx>,
+    mut callee_body: Body<'tcx>,
+) {
+    let tcx = inliner.tcx();
+    let terminator = caller_body[callsite.block].terminator.take().unwrap();
+    let TerminatorKind::Call { func, args, destination, unwind, target, .. } = terminator.kind
+    else {
+        bug!("unexpected terminator kind {:?}", terminator.kind);
+    };
+
+    let return_block = if let Some(block) = target {
+        // Prepare a new block for code that should execute when call returns. We don't use
+        // target block directly since it might have other predecessors.
+        let data = BasicBlockData::new(
+            Some(Terminator {
+                source_info: terminator.source_info,
+                kind: TerminatorKind::Goto { target: block },
+            }),
+            caller_body[block].is_cleanup,
+        );
+        Some(caller_body.basic_blocks_mut().push(data))
+    } else {
+        None
+    };
+
+    // If the call is something like `a[*i] = f(i)`, where
+    // `i : &mut usize`, then just duplicating the `a[*i]`
+    // Place could result in two different locations if `f`
+    // writes to `i`. To prevent this we need to create a temporary
+    // borrow of the place and pass the destination as `*temp` instead.
+    fn dest_needs_borrow(place: Place<'_>) -> bool {
+        for elem in place.projection.iter() {
+            match elem {
+                ProjectionElem::Deref | ProjectionElem::Index(_) => return true,
+                _ => {}
+            }
         }
 
-        // Otherwise, create a temporary for the argument.
-        trace!("creating temp for argument {:?}", arg);
-        let arg_ty = arg.ty(caller_body, self.tcx);
-        let local = self.new_call_temp(caller_body, callsite, arg_ty, return_block);
-        caller_body[callsite.block].statements.push(Statement {
-            source_info: callsite.source_info,
-            kind: StatementKind::Assign(Box::new((Place::from(local), Rvalue::Use(arg)))),
-        });
-        local
+        false
     }
 
-    /// Introduces a new temporary into the caller body that is live for the duration of the call.
-    fn new_call_temp(
-        &self,
-        caller_body: &mut Body<'tcx>,
-        callsite: &CallSite<'tcx>,
-        ty: Ty<'tcx>,
-        return_block: Option<BasicBlock>,
-    ) -> Local {
-        let local = caller_body.local_decls.push(LocalDecl::new(ty, callsite.source_info.span));
-
+    let dest = if dest_needs_borrow(destination) {
+        trace!("creating temp for return destination");
+        let dest = Rvalue::Ref(
+            tcx.lifetimes.re_erased,
+            BorrowKind::Mut { kind: MutBorrowKind::Default },
+            destination,
+        );
+        let dest_ty = dest.ty(caller_body, tcx);
+        let temp = Place::from(new_call_temp(caller_body, callsite, dest_ty, return_block));
         caller_body[callsite.block].statements.push(Statement {
             source_info: callsite.source_info,
-            kind: StatementKind::StorageLive(local),
+            kind: StatementKind::Assign(Box::new((temp, dest))),
         });
+        tcx.mk_place_deref(temp)
+    } else {
+        destination
+    };
+
+    // Always create a local to hold the destination, as `RETURN_PLACE` may appear
+    // where a full `Place` is not allowed.
+    let (remap_destination, destination_local) = if let Some(d) = dest.as_local() {
+        (false, d)
+    } else {
+        (
+            true,
+            new_call_temp(caller_body, callsite, destination.ty(caller_body, tcx).ty, return_block),
+        )
+    };
 
-        if let Some(block) = return_block {
-            caller_body[block].statements.insert(0, Statement {
+    // Copy the arguments if needed.
+    let args = make_call_args(inliner, args, callsite, caller_body, &callee_body, return_block);
+
+    let mut integrator = Integrator {
+        args: &args,
+        new_locals: Local::new(caller_body.local_decls.len())..,
+        new_scopes: SourceScope::new(caller_body.source_scopes.len())..,
+        new_blocks: BasicBlock::new(caller_body.basic_blocks.len())..,
+        destination: destination_local,
+        callsite_scope: caller_body.source_scopes[callsite.source_info.scope].clone(),
+        callsite,
+        cleanup_block: unwind,
+        in_cleanup_block: false,
+        return_block,
+        tcx,
+        always_live_locals: DenseBitSet::new_filled(callee_body.local_decls.len()),
+    };
+
+    // Map all `Local`s, `SourceScope`s and `BasicBlock`s to new ones
+    // (or existing ones, in a few special cases) in the caller.
+    integrator.visit_body(&mut callee_body);
+
+    // If there are any locals without storage markers, give them storage only for the
+    // duration of the call.
+    for local in callee_body.vars_and_temps_iter() {
+        if integrator.always_live_locals.contains(local) {
+            let new_local = integrator.map_local(local);
+            caller_body[callsite.block].statements.push(Statement {
+                source_info: callsite.source_info,
+                kind: StatementKind::StorageLive(new_local),
+            });
+        }
+    }
+    if let Some(block) = return_block {
+        // To avoid repeated O(n) insert, push any new statements to the end and rotate
+        // the slice once.
+        let mut n = 0;
+        if remap_destination {
+            caller_body[block].statements.push(Statement {
                 source_info: callsite.source_info,
-                kind: StatementKind::StorageDead(local),
+                kind: StatementKind::Assign(Box::new((
+                    dest,
+                    Rvalue::Use(Operand::Move(destination_local.into())),
+                ))),
             });
+            n += 1;
+        }
+        for local in callee_body.vars_and_temps_iter().rev() {
+            if integrator.always_live_locals.contains(local) {
+                let new_local = integrator.map_local(local);
+                caller_body[block].statements.push(Statement {
+                    source_info: callsite.source_info,
+                    kind: StatementKind::StorageDead(new_local),
+                });
+                n += 1;
+            }
         }
+        caller_body[block].statements.rotate_right(n);
+    }
 
-        local
+    // Insert all of the (mapped) parts of the callee body into the caller.
+    caller_body.local_decls.extend(callee_body.drain_vars_and_temps());
+    caller_body.source_scopes.append(&mut callee_body.source_scopes);
+    if tcx
+        .sess
+        .opts
+        .unstable_opts
+        .inline_mir_preserve_debug
+        .unwrap_or(tcx.sess.opts.debuginfo != DebugInfo::None)
+    {
+        // Note that we need to preserve these in the standard library so that
+        // people working on rust can build with or without debuginfo while
+        // still getting consistent results from the mir-opt tests.
+        caller_body.var_debug_info.append(&mut callee_body.var_debug_info);
     }
+    caller_body.basic_blocks_mut().append(callee_body.basic_blocks_mut());
+
+    caller_body[callsite.block].terminator = Some(Terminator {
+        source_info: callsite.source_info,
+        kind: TerminatorKind::Goto { target: integrator.map_block(START_BLOCK) },
+    });
+
+    // Copy required constants from the callee_body into the caller_body. Although we are only
+    // pushing unevaluated consts to `required_consts`, here they may have been evaluated
+    // because we are calling `instantiate_and_normalize_erasing_regions` -- so we filter again.
+    caller_body.required_consts.as_mut().unwrap().extend(
+        callee_body.required_consts().into_iter().filter(|ct| ct.const_.is_required_const()),
+    );
+    // Now that we incorporated the callee's `required_consts`, we can remove the callee from
+    // `mentioned_items` -- but we have to take their `mentioned_items` in return. This does
+    // some extra work here to save the monomorphization collector work later. It helps a lot,
+    // since monomorphization can avoid a lot of work when the "mentioned items" are similar to
+    // the actually used items. By doing this we can entirely avoid visiting the callee!
+    // We need to reconstruct the `required_item` for the callee so that we can find and
+    // remove it.
+    let callee_item = MentionedItem::Fn(func.ty(caller_body, tcx));
+    let caller_mentioned_items = caller_body.mentioned_items.as_mut().unwrap();
+    if let Some(idx) = caller_mentioned_items.iter().position(|item| item.node == callee_item) {
+        // We found the callee, so remove it and add its items instead.
+        caller_mentioned_items.remove(idx);
+        caller_mentioned_items.extend(callee_body.mentioned_items());
+    } else {
+        // If we can't find the callee, there's no point in adding its items. Probably it
+        // already got removed by being inlined elsewhere in the same function, so we already
+        // took its items.
+    }
+}
+
+fn make_call_args<'tcx, I: Inliner<'tcx>>(
+    inliner: &I,
+    args: Box<[Spanned<Operand<'tcx>>]>,
+    callsite: &CallSite<'tcx>,
+    caller_body: &mut Body<'tcx>,
+    callee_body: &Body<'tcx>,
+    return_block: Option<BasicBlock>,
+) -> Box<[Local]> {
+    let tcx = inliner.tcx();
+
+    // There is a bit of a mismatch between the *caller* of a closure and the *callee*.
+    // The caller provides the arguments wrapped up in a tuple:
+    //
+    //     tuple_tmp = (a, b, c)
+    //     Fn::call(closure_ref, tuple_tmp)
+    //
+    // meanwhile the closure body expects the arguments (here, `a`, `b`, and `c`)
+    // as distinct arguments. (This is the "rust-call" ABI hack.) Normally, codegen has
+    // the job of unpacking this tuple. But here, we are codegen. =) So we want to create
+    // a vector like
+    //
+    //     [closure_ref, tuple_tmp.0, tuple_tmp.1, tuple_tmp.2]
+    //
+    // Except for one tiny wrinkle: we don't actually want `tuple_tmp.0`. It's more convenient
+    // if we "spill" that into *another* temporary, so that we can map the argument
+    // variable in the callee MIR directly to an argument variable on our side.
+    // So we introduce temporaries like:
+    //
+    //     tmp0 = tuple_tmp.0
+    //     tmp1 = tuple_tmp.1
+    //     tmp2 = tuple_tmp.2
+    //
+    // and the vector is `[closure_ref, tmp0, tmp1, tmp2]`.
+    if callsite.fn_sig.abi() == ExternAbi::RustCall && callee_body.spread_arg.is_none() {
+        // FIXME(edition_2024): switch back to a normal method call.
+        let mut args = <_>::into_iter(args);
+        let self_ = create_temp_if_necessary(
+            inliner,
+            args.next().unwrap().node,
+            callsite,
+            caller_body,
+            return_block,
+        );
+        let tuple = create_temp_if_necessary(
+            inliner,
+            args.next().unwrap().node,
+            callsite,
+            caller_body,
+            return_block,
+        );
+        assert!(args.next().is_none());
+
+        let tuple = Place::from(tuple);
+        let ty::Tuple(tuple_tys) = tuple.ty(caller_body, tcx).ty.kind() else {
+            bug!("Closure arguments are not passed as a tuple");
+        };
+
+        // The `closure_ref` in our example above.
+        let closure_ref_arg = iter::once(self_);
+
+        // The `tmp0`, `tmp1`, and `tmp2` in our example above.
+        let tuple_tmp_args = tuple_tys.iter().enumerate().map(|(i, ty)| {
+            // This is e.g., `tuple_tmp.0` in our example above.
+            let tuple_field = Operand::Move(tcx.mk_place_field(tuple, FieldIdx::new(i), ty));
+
+            // Spill to a local to make e.g., `tmp0`.
+            create_temp_if_necessary(inliner, tuple_field, callsite, caller_body, return_block)
+        });
+
+        closure_ref_arg.chain(tuple_tmp_args).collect()
+    } else {
+        // FIXME(edition_2024): switch back to a normal method call.
+        <_>::into_iter(args)
+            .map(|a| create_temp_if_necessary(inliner, a.node, callsite, caller_body, return_block))
+            .collect()
+    }
+}
+
+/// If `arg` is already a temporary, returns it. Otherwise, introduces a fresh temporary `T` and an
+/// instruction `T = arg`, and returns `T`.
+fn create_temp_if_necessary<'tcx, I: Inliner<'tcx>>(
+    inliner: &I,
+    arg: Operand<'tcx>,
+    callsite: &CallSite<'tcx>,
+    caller_body: &mut Body<'tcx>,
+    return_block: Option<BasicBlock>,
+) -> Local {
+    // Reuse the operand if it is a moved temporary.
+    if let Operand::Move(place) = &arg
+        && let Some(local) = place.as_local()
+        && caller_body.local_kind(local) == LocalKind::Temp
+    {
+        return local;
+    }
+
+    // Otherwise, create a temporary for the argument.
+    trace!("creating temp for argument {:?}", arg);
+    let arg_ty = arg.ty(caller_body, inliner.tcx());
+    let local = new_call_temp(caller_body, callsite, arg_ty, return_block);
+    caller_body[callsite.block].statements.push(Statement {
+        source_info: callsite.source_info,
+        kind: StatementKind::Assign(Box::new((Place::from(local), Rvalue::Use(arg)))),
+    });
+    local
+}
+
+/// Introduces a new temporary into the caller body that is live for the duration of the call.
+fn new_call_temp<'tcx>(
+    caller_body: &mut Body<'tcx>,
+    callsite: &CallSite<'tcx>,
+    ty: Ty<'tcx>,
+    return_block: Option<BasicBlock>,
+) -> Local {
+    let local = caller_body.local_decls.push(LocalDecl::new(ty, callsite.source_info.span));
+
+    caller_body[callsite.block].statements.push(Statement {
+        source_info: callsite.source_info,
+        kind: StatementKind::StorageLive(local),
+    });
+
+    if let Some(block) = return_block {
+        caller_body[block].statements.insert(0, Statement {
+            source_info: callsite.source_info,
+            kind: StatementKind::StorageDead(local),
+        });
+    }
+
+    local
 }
 
 /**
@@ -894,7 +1127,7 @@ struct Integrator<'a, 'tcx> {
     in_cleanup_block: bool,
     return_block: Option<BasicBlock>,
     tcx: TyCtxt<'tcx>,
-    always_live_locals: BitSet<Local>,
+    always_live_locals: DenseBitSet<Local>,
 }
 
 impl Integrator<'_, '_> {
diff --git a/compiler/rustc_mir_transform/src/instsimplify.rs b/compiler/rustc_mir_transform/src/instsimplify.rs
index 1a65affe812..20e2e3e8ba2 100644
--- a/compiler/rustc_mir_transform/src/instsimplify.rs
+++ b/compiler/rustc_mir_transform/src/instsimplify.rs
@@ -48,6 +48,8 @@ impl<'tcx> crate::MirPass<'tcx> for InstSimplify {
                         ctx.simplify_ref_deref(rvalue);
                         ctx.simplify_ptr_aggregate(rvalue);
                         ctx.simplify_cast(rvalue);
+                        ctx.simplify_repeated_aggregate(rvalue);
+                        ctx.simplify_repeat_once(rvalue);
                     }
                     _ => {}
                 }
@@ -68,6 +70,35 @@ struct InstSimplifyContext<'a, 'tcx> {
 }
 
 impl<'tcx> InstSimplifyContext<'_, 'tcx> {
+    /// Transform aggregates like [0, 0, 0, 0, 0] into [0; 5].
+    /// GVN can also do this optimization, but GVN is only run at mir-opt-level 2 so having this in
+    /// InstSimplify helps unoptimized builds.
+    fn simplify_repeated_aggregate(&self, rvalue: &mut Rvalue<'tcx>) {
+        let Rvalue::Aggregate(box AggregateKind::Array(_), fields) = rvalue else {
+            return;
+        };
+        if fields.len() < 5 {
+            return;
+        }
+        let first = &fields[rustc_abi::FieldIdx::ZERO];
+        let Operand::Constant(first) = first else {
+            return;
+        };
+        let Ok(first_val) = first.const_.eval(self.tcx, self.typing_env, first.span) else {
+            return;
+        };
+        if fields.iter().all(|field| {
+            let Operand::Constant(field) = field else {
+                return false;
+            };
+            let field = field.const_.eval(self.tcx, self.typing_env, field.span);
+            field == Ok(first_val)
+        }) {
+            let len = ty::Const::from_target_usize(self.tcx, fields.len().try_into().unwrap());
+            *rvalue = Rvalue::Repeat(Operand::Constant(first.clone()), len);
+        }
+    }
+
     /// Transform boolean comparisons into logical operations.
     fn simplify_bool_cmp(&self, rvalue: &mut Rvalue<'tcx>) {
         match rvalue {
@@ -173,33 +204,22 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> {
                     *kind = CastKind::IntToInt;
                     return;
                 }
-
-                // Transmuting a transparent struct/union to a field's type is a projection
-                if let ty::Adt(adt_def, args) = operand_ty.kind()
-                    && adt_def.repr().transparent()
-                    && (adt_def.is_struct() || adt_def.is_union())
-                    && let Some(place) = operand.place()
-                {
-                    let variant = adt_def.non_enum_variant();
-                    for (i, field) in variant.fields.iter_enumerated() {
-                        let field_ty = field.ty(self.tcx, args);
-                        if field_ty == *cast_ty {
-                            let place = place
-                                .project_deeper(&[ProjectionElem::Field(i, *cast_ty)], self.tcx);
-                            let operand = if operand.is_move() {
-                                Operand::Move(place)
-                            } else {
-                                Operand::Copy(place)
-                            };
-                            *rvalue = Rvalue::Use(operand);
-                            return;
-                        }
-                    }
-                }
             }
         }
     }
 
+    /// Simplify `[x; 1]` to just `[x]`.
+    fn simplify_repeat_once(&self, rvalue: &mut Rvalue<'tcx>) {
+        if let Rvalue::Repeat(operand, count) = rvalue
+            && let Some(1) = count.try_to_target_usize(self.tcx)
+        {
+            *rvalue = Rvalue::Aggregate(
+                Box::new(AggregateKind::Array(operand.ty(self.local_decls, self.tcx))),
+                [operand.clone()].into(),
+            );
+        }
+    }
+
     fn simplify_primitive_clone(
         &self,
         terminator: &mut Terminator<'tcx>,
diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs
index 8feb90ff7a0..c73a03489c5 100644
--- a/compiler/rustc_mir_transform/src/jump_threading.rs
+++ b/compiler/rustc_mir_transform/src/jump_threading.rs
@@ -40,7 +40,7 @@ use rustc_const_eval::const_eval::DummyMachine;
 use rustc_const_eval::interpret::{ImmTy, Immediate, InterpCx, OpTy, Projectable};
 use rustc_data_structures::fx::FxHashSet;
 use rustc_index::IndexVec;
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_middle::bug;
 use rustc_middle::mir::interpret::Scalar;
 use rustc_middle::mir::visit::Visitor;
@@ -121,7 +121,7 @@ struct TOFinder<'a, 'tcx> {
     ecx: InterpCx<'tcx, DummyMachine>,
     body: &'a Body<'tcx>,
     map: Map<'tcx>,
-    loop_headers: BitSet<BasicBlock>,
+    loop_headers: DenseBitSet<BasicBlock>,
     /// We use an arena to avoid cloning the slices when cloning `state`.
     arena: &'a DroplessArena,
     opportunities: Vec<ThreadingOpportunity>,
@@ -832,8 +832,8 @@ enum Update {
 /// at least a predecessor which it dominates. This definition is only correct for reducible CFGs.
 /// But if the CFG is already irreducible, there is no point in trying much harder.
 /// is already irreducible.
-fn loop_headers(body: &Body<'_>) -> BitSet<BasicBlock> {
-    let mut loop_headers = BitSet::new_empty(body.basic_blocks.len());
+fn loop_headers(body: &Body<'_>) -> DenseBitSet<BasicBlock> {
+    let mut loop_headers = DenseBitSet::new_empty(body.basic_blocks.len());
     let dominators = body.basic_blocks.dominators();
     // Only visit reachable blocks.
     for (bb, bbdata) in traversal::preorder(body) {
diff --git a/compiler/rustc_mir_transform/src/known_panics_lint.rs b/compiler/rustc_mir_transform/src/known_panics_lint.rs
index f1705d0c831..9b3a0e67295 100644
--- a/compiler/rustc_mir_transform/src/known_panics_lint.rs
+++ b/compiler/rustc_mir_transform/src/known_panics_lint.rs
@@ -13,7 +13,7 @@ use rustc_data_structures::fx::FxHashSet;
 use rustc_hir::HirId;
 use rustc_hir::def::DefKind;
 use rustc_index::IndexVec;
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_middle::bug;
 use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor};
 use rustc_middle::mir::*;
@@ -67,7 +67,7 @@ struct ConstPropagator<'mir, 'tcx> {
     tcx: TyCtxt<'tcx>,
     typing_env: ty::TypingEnv<'tcx>,
     worklist: Vec<BasicBlock>,
-    visited_blocks: BitSet<BasicBlock>,
+    visited_blocks: DenseBitSet<BasicBlock>,
     locals: IndexVec<Local, Value<'tcx>>,
     body: &'mir Body<'tcx>,
     written_only_inside_own_block_locals: FxHashSet<Local>,
@@ -190,7 +190,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
             tcx,
             typing_env,
             worklist: vec![START_BLOCK],
-            visited_blocks: BitSet::new_empty(body.basic_blocks.len()),
+            visited_blocks: DenseBitSet::new_empty(body.basic_blocks.len()),
             locals: IndexVec::from_elem_n(Value::Uninit, body.local_decls.len()),
             body,
             can_const_prop,
@@ -852,7 +852,7 @@ enum ConstPropMode {
 struct CanConstProp {
     can_const_prop: IndexVec<Local, ConstPropMode>,
     // False at the beginning. Once set, no more assignments are allowed to that local.
-    found_assignment: BitSet<Local>,
+    found_assignment: DenseBitSet<Local>,
 }
 
 impl CanConstProp {
@@ -864,7 +864,7 @@ impl CanConstProp {
     ) -> IndexVec<Local, ConstPropMode> {
         let mut cpv = CanConstProp {
             can_const_prop: IndexVec::from_elem(ConstPropMode::FullConstProp, &body.local_decls),
-            found_assignment: BitSet::new_empty(body.local_decls.len()),
+            found_assignment: DenseBitSet::new_empty(body.local_decls.len()),
         };
         for (local, val) in cpv.can_const_prop.iter_enumerated_mut() {
             let ty = body.local_decls[local].ty;
diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs
index e1fba9be5bb..9459ef99445 100644
--- a/compiler/rustc_mir_transform/src/lib.rs
+++ b/compiler/rustc_mir_transform/src/lib.rs
@@ -34,8 +34,7 @@ use rustc_middle::util::Providers;
 use rustc_middle::{bug, query, span_bug};
 use rustc_span::source_map::Spanned;
 use rustc_span::{DUMMY_SP, sym};
-use rustc_trait_selection::traits;
-use tracing::{debug, trace};
+use tracing::debug;
 
 #[macro_use]
 mod pass_manager;
@@ -141,7 +140,8 @@ declare_passes! {
     mod gvn : GVN;
     // Made public so that `mir_drops_elaborated_and_const_checked` can be overridden
     // by custom rustc drivers, running all the steps by themselves. See #114628.
-    pub mod inline : Inline;
+    pub mod inline : Inline, ForceInline;
+    mod impossible_predicates : ImpossiblePredicates;
     mod instsimplify : InstSimplify { BeforeInline, AfterSimplifyCfg };
     mod jump_threading : JumpThreading;
     mod known_panics_lint : KnownPanicsLint;
@@ -488,7 +488,9 @@ fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> &
     let is_fn_like = tcx.def_kind(def).is_fn_like();
     if is_fn_like {
         // Do not compute the mir call graph without said call graph actually being used.
-        if pm::should_run_pass(tcx, &inline::Inline) {
+        if pm::should_run_pass(tcx, &inline::Inline)
+            || inline::ForceInline::should_run_pass_for_callee(tcx, def.to_def_id())
+        {
             tcx.ensure_with_value().mir_inliner_callees(ty::InstanceKind::Item(def.to_def_id()));
         }
     }
@@ -500,50 +502,6 @@ fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> &
         body.tainted_by_errors = Some(error_reported);
     }
 
-    // Check if it's even possible to satisfy the 'where' clauses
-    // for this item.
-    //
-    // This branch will never be taken for any normal function.
-    // However, it's possible to `#!feature(trivial_bounds)]` to write
-    // a function with impossible to satisfy clauses, e.g.:
-    // `fn foo() where String: Copy {}`
-    //
-    // We don't usually need to worry about this kind of case,
-    // since we would get a compilation error if the user tried
-    // to call it. However, since we optimize even without any
-    // calls to the function, we need to make sure that it even
-    // makes sense to try to evaluate the body.
-    //
-    // If there are unsatisfiable where clauses, then all bets are
-    // off, and we just give up.
-    //
-    // We manually filter the predicates, skipping anything that's not
-    // "global". We are in a potentially generic context
-    // (e.g. we are evaluating a function without instantiating generic
-    // parameters, so this filtering serves two purposes:
-    //
-    // 1. We skip evaluating any predicates that we would
-    // never be able prove are unsatisfiable (e.g. `<T as Foo>`
-    // 2. We avoid trying to normalize predicates involving generic
-    // parameters (e.g. `<T as Foo>::MyItem`). This can confuse
-    // the normalization code (leading to cycle errors), since
-    // it's usually never invoked in this way.
-    let predicates = tcx
-        .predicates_of(body.source.def_id())
-        .predicates
-        .iter()
-        .filter_map(|(p, _)| if p.is_global() { Some(*p) } else { None });
-    if traits::impossible_predicates(tcx, traits::elaborate(tcx, predicates).collect()) {
-        trace!("found unsatisfiable predicates for {:?}", body.source);
-        // Clear the body to only contain a single `unreachable` statement.
-        let bbs = body.basic_blocks.as_mut();
-        bbs.raw.truncate(1);
-        bbs[START_BLOCK].statements.clear();
-        bbs[START_BLOCK].terminator_mut().kind = TerminatorKind::Unreachable;
-        body.var_debug_info.clear();
-        body.local_decls.raw.truncate(body.arg_count + 1);
-    }
-
     run_analysis_to_runtime_passes(tcx, &mut body);
 
     // Now that drop elaboration has been performed, we can check for
@@ -591,6 +549,7 @@ pub fn run_analysis_to_runtime_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'
 /// After this series of passes, no lifetime analysis based on borrowing can be done.
 fn run_analysis_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
     let passes: &[&dyn MirPass<'tcx>] = &[
+        &impossible_predicates::ImpossiblePredicates,
         &cleanup_post_borrowck::CleanupPostBorrowck,
         &remove_noop_landing_pads::RemoveNoopLandingPads,
         &simplify::SimplifyCfg::PostAnalysis,
@@ -664,6 +623,8 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
             // Perform instsimplify before inline to eliminate some trivial calls (like clone
             // shims).
             &instsimplify::InstSimplify::BeforeInline,
+            // Perform inlining of `#[rustc_force_inline]`-annotated callees.
+            &inline::ForceInline,
             // Perform inlining, which may add a lot of code.
             &inline::Inline,
             // Code from other crates may have storage markers, so this needs to happen after
diff --git a/compiler/rustc_mir_transform/src/lint.rs b/compiler/rustc_mir_transform/src/lint.rs
index 29e762af8de..f472c7cb493 100644
--- a/compiler/rustc_mir_transform/src/lint.rs
+++ b/compiler/rustc_mir_transform/src/lint.rs
@@ -5,7 +5,7 @@
 use std::borrow::Cow;
 
 use rustc_data_structures::fx::FxHashSet;
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_middle::mir::visit::{PlaceContext, Visitor};
 use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
@@ -43,7 +43,7 @@ struct Lint<'a, 'tcx> {
     when: String,
     body: &'a Body<'tcx>,
     is_fn_like: bool,
-    always_live_locals: &'a BitSet<Local>,
+    always_live_locals: &'a DenseBitSet<Local>,
     maybe_storage_live: ResultsCursor<'a, 'tcx, MaybeStorageLive<'a>>,
     maybe_storage_dead: ResultsCursor<'a, 'tcx, MaybeStorageDead<'a>>,
     places: FxHashSet<PlaceRef<'tcx>>,
diff --git a/compiler/rustc_mir_transform/src/lint_tail_expr_drop_order.rs b/compiler/rustc_mir_transform/src/lint_tail_expr_drop_order.rs
index e5a183bc75c..50d10883d2c 100644
--- a/compiler/rustc_mir_transform/src/lint_tail_expr_drop_order.rs
+++ b/compiler/rustc_mir_transform/src/lint_tail_expr_drop_order.rs
@@ -351,6 +351,11 @@ pub(crate) fn run_lint<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId, body: &Body<
     {
         return;
     }
+
+    // FIXME(typing_env): This should be able to reveal the opaques local to the
+    // body using the typeck results.
+    let typing_env = ty::TypingEnv::non_body_analysis(tcx, def_id);
+
     // ## About BIDs in blocks ##
     // Track the set of blocks that contain a backwards-incompatible drop (BID)
     // and, for each block, the vector of locations.
@@ -358,7 +363,7 @@ pub(crate) fn run_lint<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId, body: &Body<
     // We group them per-block because they tend to scheduled in the same drop ladder block.
     let mut bid_per_block = IndexMap::default();
     let mut bid_places = UnordSet::new();
-    let typing_env = ty::TypingEnv::post_analysis(tcx, def_id);
+
     let mut ty_dropped_components = UnordMap::default();
     for (block, data) in body.basic_blocks.iter_enumerated() {
         for (statement_index, stmt) in data.statements.iter().enumerate() {
@@ -686,7 +691,7 @@ impl Subdiagnostic for LocalLabel<'_> {
         for dtor in self.destructors {
             dtor.add_to_diag_with(diag, f);
         }
-        let msg = f(diag, crate::fluent_generated::mir_transform_label_local_epilogue.into());
+        let msg = f(diag, crate::fluent_generated::mir_transform_label_local_epilogue);
         diag.span_label(self.span, msg);
     }
 }
diff --git a/compiler/rustc_mir_transform/src/multiple_return_terminators.rs b/compiler/rustc_mir_transform/src/multiple_return_terminators.rs
index a9227524ce5..6dfa14d6b52 100644
--- a/compiler/rustc_mir_transform/src/multiple_return_terminators.rs
+++ b/compiler/rustc_mir_transform/src/multiple_return_terminators.rs
@@ -1,7 +1,7 @@
 //! This pass removes jumps to basic blocks containing only a return, and replaces them with a
 //! return instead.
 
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
 
@@ -16,7 +16,7 @@ impl<'tcx> crate::MirPass<'tcx> for MultipleReturnTerminators {
 
     fn run_pass(&self, _: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         // find basic blocks with no statement and a return terminator
-        let mut bbs_simple_returns = BitSet::new_empty(body.basic_blocks.len());
+        let mut bbs_simple_returns = DenseBitSet::new_empty(body.basic_blocks.len());
         let bbs = body.basic_blocks_mut();
         for idx in bbs.indices() {
             if bbs[idx].statements.is_empty()
diff --git a/compiler/rustc_mir_transform/src/nrvo.rs b/compiler/rustc_mir_transform/src/nrvo.rs
index cd026ed6806..35872de3852 100644
--- a/compiler/rustc_mir_transform/src/nrvo.rs
+++ b/compiler/rustc_mir_transform/src/nrvo.rs
@@ -1,7 +1,7 @@
 //! See the docs for [`RenameReturnPlace`].
 
 use rustc_hir::Mutability;
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_middle::bug;
 use rustc_middle::mir::visit::{MutVisitor, NonUseContext, PlaceContext, Visitor};
 use rustc_middle::mir::{self, BasicBlock, Local, Location};
@@ -116,7 +116,7 @@ fn local_eligible_for_nrvo(body: &mir::Body<'_>) -> Option<Local> {
 
 fn find_local_assigned_to_return_place(start: BasicBlock, body: &mir::Body<'_>) -> Option<Local> {
     let mut block = start;
-    let mut seen = BitSet::new_empty(body.basic_blocks.len());
+    let mut seen = DenseBitSet::new_empty(body.basic_blocks.len());
 
     // Iterate as long as `block` has exactly one predecessor that we have not yet visited.
     while seen.insert(block) {
diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs
index 8a45ce0762d..c3f0a989ce1 100644
--- a/compiler/rustc_mir_transform/src/pass_manager.rs
+++ b/compiler/rustc_mir_transform/src/pass_manager.rs
@@ -79,6 +79,12 @@ pub(super) trait MirPass<'tcx> {
         true
     }
 
+    /// Returns `true` if this pass can be overridden by `-Zenable-mir-passes`. This should be
+    /// true for basically every pass other than those that are necessary for correctness.
+    fn can_be_overridden(&self) -> bool {
+        true
+    }
+
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>);
 
     fn is_mir_dump_enabled(&self) -> bool {
@@ -176,6 +182,10 @@ where
 {
     let name = pass.name();
 
+    if !pass.can_be_overridden() {
+        return pass.is_enabled(tcx.sess);
+    }
+
     let overridden_passes = &tcx.sess.opts.unstable_opts.mir_enable_passes;
     let overridden =
         overridden_passes.iter().rev().find(|(s, _)| s == &*name).map(|(_name, polarity)| {
diff --git a/compiler/rustc_mir_transform/src/prettify.rs b/compiler/rustc_mir_transform/src/prettify.rs
index 937c207776b..51abd4da86e 100644
--- a/compiler/rustc_mir_transform/src/prettify.rs
+++ b/compiler/rustc_mir_transform/src/prettify.rs
@@ -4,7 +4,7 @@
 //! (`-Zmir-enable-passes=+ReorderBasicBlocks,+ReorderLocals`)
 //! to make the MIR easier to read for humans.
 
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_index::{IndexSlice, IndexVec};
 use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor};
 use rustc_middle::mir::*;
@@ -51,8 +51,10 @@ impl<'tcx> crate::MirPass<'tcx> for ReorderLocals {
     }
 
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-        let mut finder =
-            LocalFinder { map: IndexVec::new(), seen: BitSet::new_empty(body.local_decls.len()) };
+        let mut finder = LocalFinder {
+            map: IndexVec::new(),
+            seen: DenseBitSet::new_empty(body.local_decls.len()),
+        };
 
         // We can't reorder the return place or the arguments
         for local in (0..=body.arg_count).map(Local::from_usize) {
@@ -113,7 +115,7 @@ impl<'tcx> MutVisitor<'tcx> for BasicBlockUpdater<'tcx> {
 
 struct LocalFinder {
     map: IndexVec<Local, Local>,
-    seen: BitSet<Local>,
+    seen: DenseBitSet<Local>,
 }
 
 impl LocalFinder {
diff --git a/compiler/rustc_mir_transform/src/ref_prop.rs b/compiler/rustc_mir_transform/src/ref_prop.rs
index 96bcdfa6fac..95b05f94270 100644
--- a/compiler/rustc_mir_transform/src/ref_prop.rs
+++ b/compiler/rustc_mir_transform/src/ref_prop.rs
@@ -2,7 +2,7 @@ use std::borrow::Cow;
 
 use rustc_data_structures::fx::FxHashSet;
 use rustc_index::IndexVec;
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_middle::bug;
 use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
@@ -132,7 +132,7 @@ fn compute_replacement<'tcx>(
     let mut targets = IndexVec::from_elem(Value::Unknown, &body.local_decls);
     // Set of locals for which we will remove their storage statement. This is useful for
     // reborrowed references.
-    let mut storage_to_remove = BitSet::new_empty(body.local_decls.len());
+    let mut storage_to_remove = DenseBitSet::new_empty(body.local_decls.len());
 
     let fully_replacable_locals = fully_replacable_locals(ssa);
 
@@ -324,8 +324,8 @@ fn compute_replacement<'tcx>(
 ///
 /// We consider a local to be replacable iff it's only used in a `Deref` projection `*_local` or
 /// non-use position (like storage statements and debuginfo).
-fn fully_replacable_locals(ssa: &SsaLocals) -> BitSet<Local> {
-    let mut replacable = BitSet::new_empty(ssa.num_locals());
+fn fully_replacable_locals(ssa: &SsaLocals) -> DenseBitSet<Local> {
+    let mut replacable = DenseBitSet::new_empty(ssa.num_locals());
 
     // First pass: for each local, whether its uses can be fully replaced.
     for local in ssa.locals() {
@@ -344,7 +344,7 @@ fn fully_replacable_locals(ssa: &SsaLocals) -> BitSet<Local> {
 struct Replacer<'tcx> {
     tcx: TyCtxt<'tcx>,
     targets: IndexVec<Local, Value<'tcx>>,
-    storage_to_remove: BitSet<Local>,
+    storage_to_remove: DenseBitSet<Local>,
     allowed_replacements: FxHashSet<(Local, Location)>,
     any_replacement: bool,
 }
diff --git a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs
index fd49e956f43..76a3edfe0be 100644
--- a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs
+++ b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs
@@ -1,4 +1,4 @@
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_middle::mir::patch::MirPatch;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
@@ -40,7 +40,7 @@ impl<'tcx> crate::MirPass<'tcx> for RemoveNoopLandingPads {
 
         let mut jumps_folded = 0;
         let mut landing_pads_removed = 0;
-        let mut nop_landing_pads = BitSet::new_empty(body.basic_blocks.len());
+        let mut nop_landing_pads = DenseBitSet::new_empty(body.basic_blocks.len());
 
         // This is a post-order traversal, so that if A post-dominates B
         // then A will be visited before B.
@@ -81,7 +81,7 @@ impl RemoveNoopLandingPads {
         &self,
         bb: BasicBlock,
         body: &Body<'_>,
-        nop_landing_pads: &BitSet<BasicBlock>,
+        nop_landing_pads: &DenseBitSet<BasicBlock>,
     ) -> bool {
         for stmt in &body[bb].statements {
             match &stmt.kind {
diff --git a/compiler/rustc_mir_transform/src/remove_zsts.rs b/compiler/rustc_mir_transform/src/remove_zsts.rs
index 64e183bcbc0..55e5701bd0a 100644
--- a/compiler/rustc_mir_transform/src/remove_zsts.rs
+++ b/compiler/rustc_mir_transform/src/remove_zsts.rs
@@ -36,31 +36,39 @@ struct Replacer<'a, 'tcx> {
 }
 
 /// A cheap, approximate check to avoid unnecessary `layout_of` calls.
-fn maybe_zst(ty: Ty<'_>) -> bool {
+///
+/// `Some(true)` is definitely ZST; `Some(false)` is definitely *not* ZST.
+///
+/// `None` may or may not be, and must check `layout_of` to be sure.
+fn trivially_zst<'tcx>(ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> Option<bool> {
     match ty.kind() {
-        // maybe ZST (could be more precise)
-        ty::Adt(..)
-        | ty::Array(..)
-        | ty::Closure(..)
-        | ty::CoroutineClosure(..)
-        | ty::Tuple(..)
-        | ty::Alias(ty::Opaque, ..) => true,
         // definitely ZST
-        ty::FnDef(..) | ty::Never => true,
-        // unreachable or can't be ZST
-        _ => false,
+        ty::FnDef(..) | ty::Never => Some(true),
+        ty::Tuple(fields) if fields.is_empty() => Some(true),
+        ty::Array(_ty, len) if let Some(0) = len.try_to_target_usize(tcx) => Some(true),
+        // clearly not ZST
+        ty::Bool
+        | ty::Char
+        | ty::Int(..)
+        | ty::Uint(..)
+        | ty::Float(..)
+        | ty::RawPtr(..)
+        | ty::Ref(..)
+        | ty::FnPtr(..) => Some(false),
+        // check `layout_of` to see (including unreachable things we won't actually see)
+        _ => None,
     }
 }
 
 impl<'tcx> Replacer<'_, 'tcx> {
     fn known_to_be_zst(&self, ty: Ty<'tcx>) -> bool {
-        if !maybe_zst(ty) {
-            return false;
+        if let Some(is_zst) = trivially_zst(ty, self.tcx) {
+            is_zst
+        } else {
+            self.tcx
+                .layout_of(self.typing_env.as_query_input(ty))
+                .is_ok_and(|layout| layout.is_zst())
         }
-        let Ok(layout) = self.tcx.layout_of(self.typing_env.as_query_input(ty)) else {
-            return false;
-        };
-        layout.is_zst()
     }
 
     fn make_zst(&self, ty: Ty<'tcx>) -> ConstOperand<'tcx> {
diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs
index 722da3c420d..4648ec33c93 100644
--- a/compiler/rustc_mir_transform/src/shim.rs
+++ b/compiler/rustc_mir_transform/src/shim.rs
@@ -20,7 +20,7 @@ use rustc_span::{DUMMY_SP, Span};
 use tracing::{debug, instrument};
 
 use crate::{
-    abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, deref_separator,
+    abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, deref_separator, inline,
     instsimplify, mentioned_items, pass_manager as pm, remove_noop_landing_pads, simplify,
 };
 
@@ -155,6 +155,8 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body<
             &remove_noop_landing_pads::RemoveNoopLandingPads,
             &simplify::SimplifyCfg::MakeShim,
             &instsimplify::InstSimplify::BeforeInline,
+            // Perform inlining of `#[rustc_force_inline]`-annotated callees.
+            &inline::ForceInline,
             &abort_unwinding_calls::AbortUnwindingCalls,
             &add_call_guards::CriticalCallEdges,
         ],
diff --git a/compiler/rustc_mir_transform/src/single_use_consts.rs b/compiler/rustc_mir_transform/src/single_use_consts.rs
index 277a33c0311..10b3c0ae94f 100644
--- a/compiler/rustc_mir_transform/src/single_use_consts.rs
+++ b/compiler/rustc_mir_transform/src/single_use_consts.rs
@@ -1,5 +1,5 @@
 use rustc_index::IndexVec;
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_middle::bug;
 use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor};
 use rustc_middle::mir::*;
@@ -28,9 +28,9 @@ impl<'tcx> crate::MirPass<'tcx> for SingleUseConsts {
 
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         let mut finder = SingleUseConstsFinder {
-            ineligible_locals: BitSet::new_empty(body.local_decls.len()),
+            ineligible_locals: DenseBitSet::new_empty(body.local_decls.len()),
             locations: IndexVec::from_elem(LocationPair::new(), &body.local_decls),
-            locals_in_debug_info: BitSet::new_empty(body.local_decls.len()),
+            locals_in_debug_info: DenseBitSet::new_empty(body.local_decls.len()),
         };
 
         finder.ineligible_locals.insert_range(..=Local::from_usize(body.arg_count));
@@ -96,9 +96,9 @@ impl LocationPair {
 }
 
 struct SingleUseConstsFinder {
-    ineligible_locals: BitSet<Local>,
+    ineligible_locals: DenseBitSet<Local>,
     locations: IndexVec<Local, LocationPair>,
-    locals_in_debug_info: BitSet<Local>,
+    locals_in_debug_info: DenseBitSet<Local>,
 }
 
 impl<'tcx> Visitor<'tcx> for SingleUseConstsFinder {
diff --git a/compiler/rustc_mir_transform/src/sroa.rs b/compiler/rustc_mir_transform/src/sroa.rs
index 52b9ec1e0a3..d54ea3feab6 100644
--- a/compiler/rustc_mir_transform/src/sroa.rs
+++ b/compiler/rustc_mir_transform/src/sroa.rs
@@ -2,7 +2,7 @@ use rustc_abi::{FIRST_VARIANT, FieldIdx};
 use rustc_data_structures::flat_map_in_place::FlatMapInPlace;
 use rustc_hir::LangItem;
 use rustc_index::IndexVec;
-use rustc_index::bit_set::{BitSet, GrowableBitSet};
+use rustc_index::bit_set::{DenseBitSet, GrowableBitSet};
 use rustc_middle::bug;
 use rustc_middle::mir::patch::MirPatch;
 use rustc_middle::mir::visit::*;
@@ -60,9 +60,9 @@ impl<'tcx> crate::MirPass<'tcx> for ScalarReplacementOfAggregates {
 fn escaping_locals<'tcx>(
     tcx: TyCtxt<'tcx>,
     typing_env: ty::TypingEnv<'tcx>,
-    excluded: &BitSet<Local>,
+    excluded: &DenseBitSet<Local>,
     body: &Body<'tcx>,
-) -> BitSet<Local> {
+) -> DenseBitSet<Local> {
     let is_excluded_ty = |ty: Ty<'tcx>| {
         if ty.is_union() || ty.is_enum() {
             return true;
@@ -97,7 +97,7 @@ fn escaping_locals<'tcx>(
         false
     };
 
-    let mut set = BitSet::new_empty(body.local_decls.len());
+    let mut set = DenseBitSet::new_empty(body.local_decls.len());
     set.insert_range(RETURN_PLACE..=Local::from_usize(body.arg_count));
     for (local, decl) in body.local_decls().iter_enumerated() {
         if excluded.contains(local) || is_excluded_ty(decl.ty) {
@@ -109,7 +109,7 @@ fn escaping_locals<'tcx>(
     return visitor.set;
 
     struct EscapeVisitor {
-        set: BitSet<Local>,
+        set: DenseBitSet<Local>,
     }
 
     impl<'tcx> Visitor<'tcx> for EscapeVisitor {
@@ -198,7 +198,7 @@ fn compute_flattening<'tcx>(
     tcx: TyCtxt<'tcx>,
     typing_env: ty::TypingEnv<'tcx>,
     body: &mut Body<'tcx>,
-    escaping: BitSet<Local>,
+    escaping: DenseBitSet<Local>,
 ) -> ReplacementMap<'tcx> {
     let mut fragments = IndexVec::from_elem(None, &body.local_decls);
 
@@ -226,8 +226,8 @@ fn replace_flattened_locals<'tcx>(
     tcx: TyCtxt<'tcx>,
     body: &mut Body<'tcx>,
     replacements: ReplacementMap<'tcx>,
-) -> BitSet<Local> {
-    let mut all_dead_locals = BitSet::new_empty(replacements.fragments.len());
+) -> DenseBitSet<Local> {
+    let mut all_dead_locals = DenseBitSet::new_empty(replacements.fragments.len());
     for (local, replacements) in replacements.fragments.iter_enumerated() {
         if replacements.is_some() {
             all_dead_locals.insert(local);
@@ -267,7 +267,7 @@ struct ReplacementVisitor<'tcx, 'll> {
     /// Work to do.
     replacements: &'ll ReplacementMap<'tcx>,
     /// This is used to check that we are not leaving references to replaced locals behind.
-    all_dead_locals: BitSet<Local>,
+    all_dead_locals: DenseBitSet<Local>,
     patch: MirPatch<'tcx>,
 }
 
diff --git a/compiler/rustc_mir_transform/src/ssa.rs b/compiler/rustc_mir_transform/src/ssa.rs
index 5653aef0aae..a24b3b2e80f 100644
--- a/compiler/rustc_mir_transform/src/ssa.rs
+++ b/compiler/rustc_mir_transform/src/ssa.rs
@@ -7,7 +7,7 @@
 //! of a `Freeze` local. Those can still be considered to be SSA.
 
 use rustc_data_structures::graph::dominators::Dominators;
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_index::{IndexSlice, IndexVec};
 use rustc_middle::bug;
 use rustc_middle::middle::resolve_bound_vars::Set1;
@@ -29,7 +29,7 @@ pub(super) struct SsaLocals {
     /// We ignore non-uses (Storage statements, debuginfo).
     direct_uses: IndexVec<Local, u32>,
     /// Set of SSA locals that are immutably borrowed.
-    borrowed_locals: BitSet<Local>,
+    borrowed_locals: DenseBitSet<Local>,
 }
 
 pub(super) enum AssignedValue<'a, 'tcx> {
@@ -50,7 +50,7 @@ impl SsaLocals {
         let dominators = body.basic_blocks.dominators();
 
         let direct_uses = IndexVec::from_elem(0, &body.local_decls);
-        let borrowed_locals = BitSet::new_empty(body.local_decls.len());
+        let borrowed_locals = DenseBitSet::new_empty(body.local_decls.len());
         let mut visitor = SsaVisitor {
             body,
             assignments,
@@ -202,12 +202,12 @@ impl SsaLocals {
     }
 
     /// Set of SSA locals that are immutably borrowed.
-    pub(super) fn borrowed_locals(&self) -> &BitSet<Local> {
+    pub(super) fn borrowed_locals(&self) -> &DenseBitSet<Local> {
         &self.borrowed_locals
     }
 
     /// Make a property uniform on a copy equivalence class by removing elements.
-    pub(super) fn meet_copy_equivalence(&self, property: &mut BitSet<Local>) {
+    pub(super) fn meet_copy_equivalence(&self, property: &mut DenseBitSet<Local>) {
         // Consolidate to have a local iff all its copies are.
         //
         // `copy_classes` defines equivalence classes between locals. The `local`s that recursively
@@ -241,7 +241,7 @@ struct SsaVisitor<'a, 'tcx> {
     assignment_order: Vec<Local>,
     direct_uses: IndexVec<Local, u32>,
     // Track locals that are immutably borrowed, so we can check their type is `Freeze` later.
-    borrowed_locals: BitSet<Local>,
+    borrowed_locals: DenseBitSet<Local>,
 }
 
 impl SsaVisitor<'_, '_> {
@@ -396,7 +396,7 @@ pub(crate) struct StorageLiveLocals {
 impl StorageLiveLocals {
     pub(crate) fn new(
         body: &Body<'_>,
-        always_storage_live_locals: &BitSet<Local>,
+        always_storage_live_locals: &DenseBitSet<Local>,
     ) -> StorageLiveLocals {
         let mut storage_live = IndexVec::from_elem(Set1::Empty, &body.local_decls);
         for local in always_storage_live_locals.iter() {
diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs
index a670da94fcc..414477d9004 100644
--- a/compiler/rustc_mir_transform/src/validate.rs
+++ b/compiler/rustc_mir_transform/src/validate.rs
@@ -1,10 +1,11 @@
 //! Validates the MIR to ensure that invariants are upheld.
 
 use rustc_abi::{ExternAbi, FIRST_VARIANT, Size};
+use rustc_attr_parsing::InlineAttr;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_hir::LangItem;
 use rustc_index::IndexVec;
-use rustc_index::bit_set::BitSet;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_infer::infer::TyCtxtInferExt;
 use rustc_infer::traits::{Obligation, ObligationCause};
 use rustc_middle::mir::coverage::CoverageKind;
@@ -97,7 +98,7 @@ struct CfgChecker<'a, 'tcx> {
     body: &'a Body<'tcx>,
     tcx: TyCtxt<'tcx>,
     unwind_edge_count: usize,
-    reachable_blocks: BitSet<BasicBlock>,
+    reachable_blocks: DenseBitSet<BasicBlock>,
     value_cache: FxHashSet<u128>,
     // If `false`, then the MIR must not contain `UnwindAction::Continue` or
     // `TerminatorKind::Resume`.
@@ -366,7 +367,8 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> {
                 self.check_edge(location, *target, EdgeKind::Normal);
                 self.check_unwind_edge(location, *unwind);
             }
-            TerminatorKind::Call { args, .. } | TerminatorKind::TailCall { args, .. } => {
+            TerminatorKind::Call { func, args, .. }
+            | TerminatorKind::TailCall { func, args, .. } => {
                 // FIXME(explicit_tail_calls): refactor this & add tail-call specific checks
                 if let TerminatorKind::Call { target, unwind, destination, .. } = terminator.kind {
                     if let Some(target) = target {
@@ -419,6 +421,13 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> {
                         }
                     }
                 }
+
+                if let ty::FnDef(did, ..) = func.ty(&self.body.local_decls, self.tcx).kind()
+                    && self.body.phase >= MirPhase::Runtime(RuntimePhase::Optimized)
+                    && matches!(self.tcx.codegen_fn_attrs(did).inline, InlineAttr::Force { .. })
+                {
+                    self.fail(location, "`#[rustc_force_inline]`-annotated function not inlined");
+                }
             }
             TerminatorKind::Assert { target, unwind, .. } => {
                 self.check_edge(location, *target, EdgeKind::Normal);