diff options
Diffstat (limited to 'compiler/rustc_mir_transform/src')
58 files changed, 2275 insertions, 1364 deletions
diff --git a/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs b/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs index 559df222a50..12a2fe23b14 100644 --- a/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs +++ b/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs @@ -1,6 +1,6 @@ use rustc_middle::mir::patch::MirPatch; use rustc_middle::mir::*; -use rustc_middle::ty::TyCtxt; +use rustc_middle::ty::{self, TyCtxt}; use tracing::debug; use crate::util; @@ -40,10 +40,10 @@ pub(super) struct AddMovesForPackedDrops; impl<'tcx> crate::MirPass<'tcx> for AddMovesForPackedDrops { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { debug!("add_moves_for_packed_drops({:?} @ {:?})", body.source, body.span); - - let def_id = body.source.def_id(); let mut patch = MirPatch::new(body); - let param_env = tcx.param_env(def_id); + // FIXME(#132279): This is used during the phase transition from analysis + // to runtime, so we have to manually specify the correct typing mode. + let typing_env = ty::TypingEnv::post_analysis(tcx, body.source.def_id()); for (bb, data) in body.basic_blocks.iter_enumerated() { let loc = Location { block: bb, statement_index: data.statements.len() }; @@ -51,7 +51,7 @@ impl<'tcx> crate::MirPass<'tcx> for AddMovesForPackedDrops { match terminator.kind { TerminatorKind::Drop { place, .. } - if util::is_disaligned(tcx, body, param_env, place) => + if util::is_disaligned(tcx, body, typing_env, place) => { add_move_for_packed_drop( tcx, diff --git a/compiler/rustc_mir_transform/src/check_alignment.rs b/compiler/rustc_mir_transform/src/check_alignment.rs index a9600f77c0b..1b7c89fd251 100644 --- a/compiler/rustc_mir_transform/src/check_alignment.rs +++ b/compiler/rustc_mir_transform/src/check_alignment.rs @@ -3,7 +3,7 @@ use rustc_index::IndexVec; use rustc_middle::mir::interpret::Scalar; use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; -use rustc_middle::ty::{self, ParamEnv, Ty, TyCtxt}; +use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_session::Session; use tracing::{debug, trace}; @@ -25,9 +25,9 @@ impl<'tcx> crate::MirPass<'tcx> for CheckAlignment { return; } + let typing_env = body.typing_env(tcx); let basic_blocks = body.basic_blocks.as_mut(); let local_decls = &mut body.local_decls; - let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); // This pass inserts new blocks. Each insertion changes the Location for all // statements/blocks after. Iterating or visiting the MIR in order would require updating @@ -41,7 +41,7 @@ impl<'tcx> crate::MirPass<'tcx> for CheckAlignment { let source_info = statement.source_info; let mut finder = - PointerFinder { tcx, local_decls, param_env, pointers: Vec::new() }; + PointerFinder { tcx, local_decls, typing_env, pointers: Vec::new() }; finder.visit_statement(statement, location); for (local, ty) in finder.pointers { @@ -65,7 +65,7 @@ impl<'tcx> crate::MirPass<'tcx> for CheckAlignment { struct PointerFinder<'a, 'tcx> { tcx: TyCtxt<'tcx>, local_decls: &'a mut LocalDecls<'tcx>, - param_env: ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, pointers: Vec<(Place<'tcx>, Ty<'tcx>)>, } @@ -107,7 +107,7 @@ impl<'a, 'tcx> Visitor<'tcx> for PointerFinder<'a, 'tcx> { let pointee_ty = pointer_ty.builtin_deref(true).expect("no builtin_deref for an unsafe pointer"); // Ideally we'd support this in the future, but for now we are limited to sized types. - if !pointee_ty.is_sized(self.tcx, self.param_env) { + if !pointee_ty.is_sized(self.tcx, self.typing_env) { debug!("Unsafe pointer, but pointee is not known to be sized: {:?}", pointer_ty); return; } diff --git a/compiler/rustc_mir_transform/src/check_packed_ref.rs b/compiler/rustc_mir_transform/src/check_packed_ref.rs index 1922d4fef25..e9b85ba6e9d 100644 --- a/compiler/rustc_mir_transform/src/check_packed_ref.rs +++ b/compiler/rustc_mir_transform/src/check_packed_ref.rs @@ -9,9 +9,9 @@ pub(super) struct CheckPackedRef; impl<'tcx> crate::MirLint<'tcx> for CheckPackedRef { fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) { - let param_env = tcx.param_env(body.source.def_id()); + let typing_env = body.typing_env(tcx); let source_info = SourceInfo::outermost(body.span); - let mut checker = PackedRefChecker { body, tcx, param_env, source_info }; + let mut checker = PackedRefChecker { body, tcx, typing_env, source_info }; checker.visit_body(body); } } @@ -19,7 +19,7 @@ impl<'tcx> crate::MirLint<'tcx> for CheckPackedRef { struct PackedRefChecker<'a, 'tcx> { body: &'a Body<'tcx>, tcx: TyCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, source_info: SourceInfo, } @@ -37,7 +37,8 @@ impl<'tcx> Visitor<'tcx> for PackedRefChecker<'_, 'tcx> { } fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, _location: Location) { - if context.is_borrow() && util::is_disaligned(self.tcx, self.body, self.param_env, *place) { + if context.is_borrow() && util::is_disaligned(self.tcx, self.body, self.typing_env, *place) + { let def_id = self.body.source.instance.def_id(); if let Some(impl_def_id) = self.tcx.impl_of_method(def_id) && self.tcx.is_builtin_derived(impl_def_id) diff --git a/compiler/rustc_mir_transform/src/copy_prop.rs b/compiler/rustc_mir_transform/src/copy_prop.rs index 7d6ae9843b1..9b3443d3209 100644 --- a/compiler/rustc_mir_transform/src/copy_prop.rs +++ b/compiler/rustc_mir_transform/src/copy_prop.rs @@ -28,8 +28,8 @@ impl<'tcx> crate::MirPass<'tcx> for CopyProp { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { debug!(def_id = ?body.source.def_id()); - let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); - let ssa = SsaLocals::new(tcx, body, param_env); + let typing_env = body.typing_env(tcx); + let ssa = SsaLocals::new(tcx, body, typing_env); let fully_moved = fully_moved_locals(&ssa, body); debug!(?fully_moved); diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs index 063f220501f..a3320f99cc3 100644 --- a/compiler/rustc_mir_transform/src/coroutine.rs +++ b/compiler/rustc_mir_transform/src/coroutine.rs @@ -68,14 +68,13 @@ use rustc_middle::ty::{ self, CoroutineArgs, CoroutineArgsExt, GenericArgsRef, InstanceKind, Ty, TyCtxt, TypingMode, }; use rustc_middle::{bug, span_bug}; -use rustc_mir_dataflow::Analysis; use rustc_mir_dataflow::impls::{ MaybeBorrowedLocals, MaybeLiveLocals, MaybeRequiresStorage, MaybeStorageLive, + always_storage_live_locals, }; -use rustc_mir_dataflow::storage::always_storage_live_locals; -use rustc_span::Span; +use rustc_mir_dataflow::{Analysis, Results, ResultsVisitor}; use rustc_span::def_id::{DefId, LocalDefId}; -use rustc_span::symbol::sym; +use rustc_span::{Span, sym}; use rustc_target::spec::PanicStrategy; use rustc_trait_selection::error_reporting::InferCtxtErrorExt; use rustc_trait_selection::infer::TyCtxtInferExt as _; @@ -676,12 +675,11 @@ fn locals_live_across_suspend_points<'tcx>( let mut borrowed_locals_cursor = borrowed_locals_results.clone().into_results_cursor(body); - // Calculate the MIR locals that we actually need to keep storage around - // for. - let mut requires_storage_cursor = + // Calculate the MIR locals that we need to keep storage around for. + let mut requires_storage_results = MaybeRequiresStorage::new(borrowed_locals_results.into_results_cursor(body)) - .iterate_to_fixpoint(tcx, body, None) - .into_results_cursor(body); + .iterate_to_fixpoint(tcx, body, None); + let mut requires_storage_cursor = requires_storage_results.as_results_cursor(body); // Calculate the liveness of MIR locals ignoring borrows. let mut liveness = @@ -697,8 +695,7 @@ fn locals_live_across_suspend_points<'tcx>( let loc = Location { block, statement_index: data.statements.len() }; liveness.seek_to_block_end(block); - let mut live_locals: BitSet<_> = BitSet::new_empty(body.local_decls.len()); - live_locals.union(liveness.get()); + let mut live_locals = liveness.get().clone(); if !movable { // The `liveness` variable contains the liveness of MIR locals ignoring borrows. @@ -754,7 +751,7 @@ fn locals_live_across_suspend_points<'tcx>( body, &saved_locals, always_live_locals.clone(), - requires_storage_cursor.into_results(), + requires_storage_results, ); LivenessInfo { @@ -817,9 +814,9 @@ impl ops::Deref for CoroutineSavedLocals { /// computation; see `CoroutineLayout` for more. fn compute_storage_conflicts<'mir, 'tcx>( body: &'mir Body<'tcx>, - saved_locals: &CoroutineSavedLocals, + saved_locals: &'mir CoroutineSavedLocals, always_live_locals: BitSet<Local>, - mut requires_storage: rustc_mir_dataflow::Results<'tcx, MaybeRequiresStorage<'mir, 'tcx>>, + mut requires_storage: Results<'tcx, MaybeRequiresStorage<'mir, 'tcx>>, ) -> BitMatrix<CoroutineSavedLocal, CoroutineSavedLocal> { assert_eq!(body.local_decls.len(), saved_locals.domain_size()); @@ -877,25 +874,23 @@ struct StorageConflictVisitor<'a, 'tcx> { eligible_storage_live: BitSet<Local>, } -impl<'a, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'a, 'tcx, R> +impl<'a, 'tcx> ResultsVisitor<'a, 'tcx, MaybeRequiresStorage<'a, 'tcx>> for StorageConflictVisitor<'a, 'tcx> { - type Domain = BitSet<Local>; - - fn visit_statement_before_primary_effect( + fn visit_after_early_statement_effect( &mut self, - _results: &mut R, - state: &Self::Domain, + _results: &mut Results<'tcx, MaybeRequiresStorage<'a, 'tcx>>, + state: &BitSet<Local>, _statement: &'a Statement<'tcx>, loc: Location, ) { self.apply_state(state, loc); } - fn visit_terminator_before_primary_effect( + fn visit_after_early_terminator_effect( &mut self, - _results: &mut R, - state: &Self::Domain, + _results: &mut Results<'tcx, MaybeRequiresStorage<'a, 'tcx>>, + state: &BitSet<Local>, _terminator: &'a Terminator<'tcx>, loc: Location, ) { @@ -1071,11 +1066,9 @@ fn elaborate_coroutine_drops<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // Note that `elaborate_drops` only drops the upvars of a coroutine, and // this is ok because `open_drop` can only be reached within that own // coroutine's resume function. + let typing_env = body.typing_env(tcx); - let def_id = body.source.def_id(); - let param_env = tcx.param_env(def_id); - - let mut elaborator = DropShimElaborator { body, patch: MirPatch::new(body), tcx, param_env }; + let mut elaborator = DropShimElaborator { body, patch: MirPatch::new(body), tcx, typing_env }; for (block, block_data) in body.basic_blocks.iter_enumerated() { let (target, unwind, source_info) = match block_data.terminator() { @@ -1206,9 +1199,9 @@ fn insert_panic_block<'tcx>( insert_term_block(body, kind) } -fn can_return<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, param_env: ty::ParamEnv<'tcx>) -> bool { +fn can_return<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, typing_env: ty::TypingEnv<'tcx>) -> bool { // Returning from a function with an uninhabited return type is undefined behavior. - if body.return_ty().is_privately_uninhabited(tcx, param_env) { + if body.return_ty().is_privately_uninhabited(tcx, typing_env) { return false; } @@ -1629,7 +1622,7 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { // `storage_liveness` tells us which locals have live storage at suspension points let (remap, layout, storage_liveness) = compute_layout(liveness_info, body); - let can_return = can_return(tcx, body, tcx.param_env(body.source.def_id())); + let can_return = can_return(tcx, body, body.typing_env(tcx)); // Run the transformation which converts Places from Local to coroutine struct // accesses for locals in `remap`. @@ -1776,6 +1769,7 @@ impl<'tcx> Visitor<'tcx> for EnsureCoroutineFieldAssignmentsNeverAlias<'_> { | StatementKind::Coverage(..) | StatementKind::Intrinsic(..) | StatementKind::ConstEvalCounter + | StatementKind::BackwardIncompatibleDropHint { .. } | StatementKind::Nop => {} } } diff --git a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs index 36eb435c63a..9f5bb015fa3 100644 --- a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs +++ b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs @@ -3,8 +3,6 @@ //! //! Consider an async closure like: //! ```rust -//! #![feature(async_closure)] -//! //! let x = vec![1, 2, 3]; //! //! let closure = async move || { @@ -80,7 +78,7 @@ use rustc_middle::hir::place::{Projection, ProjectionKind}; use rustc_middle::mir::visit::MutVisitor; use rustc_middle::mir::{self, dump_mir}; use rustc_middle::ty::{self, InstanceKind, Ty, TyCtxt, TypeVisitableExt}; -use rustc_span::symbol::kw; +use rustc_span::kw; pub(crate) fn coroutine_by_move_body_def_id<'tcx>( tcx: TyCtxt<'tcx>, diff --git a/compiler/rustc_mir_transform/src/cost_checker.rs b/compiler/rustc_mir_transform/src/cost_checker.rs index 59b403538a3..b23d8b9e737 100644 --- a/compiler/rustc_mir_transform/src/cost_checker.rs +++ b/compiler/rustc_mir_transform/src/cost_checker.rs @@ -1,7 +1,7 @@ use rustc_middle::bug; use rustc_middle::mir::visit::*; use rustc_middle::mir::*; -use rustc_middle::ty::{self, ParamEnv, Ty, TyCtxt}; +use rustc_middle::ty::{self, Ty, TyCtxt}; const INSTR_COST: usize = 5; const CALL_PENALTY: usize = 25; @@ -14,7 +14,7 @@ const CONST_SWITCH_BONUS: usize = 10; #[derive(Clone)] pub(super) struct CostChecker<'b, 'tcx> { tcx: TyCtxt<'tcx>, - param_env: ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, penalty: usize, bonus: usize, callee_body: &'b Body<'tcx>, @@ -24,11 +24,11 @@ pub(super) struct CostChecker<'b, 'tcx> { impl<'b, 'tcx> CostChecker<'b, 'tcx> { pub(super) fn new( tcx: TyCtxt<'tcx>, - param_env: ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, instance: Option<ty::Instance<'tcx>>, callee_body: &'b Body<'tcx>, ) -> CostChecker<'b, 'tcx> { - CostChecker { tcx, param_env, callee_body, instance, penalty: 0, bonus: 0 } + CostChecker { tcx, typing_env, callee_body, instance, penalty: 0, bonus: 0 } } /// Add function-level costs not well-represented by the block-level costs. @@ -119,7 +119,7 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { TerminatorKind::Drop { place, unwind, .. } => { // If the place doesn't actually need dropping, treat it like a regular goto. let ty = self.instantiate_ty(place.ty(self.callee_body, self.tcx).ty); - if ty.needs_drop(self.tcx, self.param_env) { + if ty.needs_drop(self.tcx, self.typing_env) { self.penalty += CALL_PENALTY; if let UnwindAction::Cleanup(_) = unwind { self.penalty += LANDINGPAD_PENALTY; diff --git a/compiler/rustc_mir_transform/src/coverage/counters.rs b/compiler/rustc_mir_transform/src/coverage/counters.rs index cf9f6981c5a..9e80f1f1c4a 100644 --- a/compiler/rustc_mir_transform/src/coverage/counters.rs +++ b/compiler/rustc_mir_transform/src/coverage/counters.rs @@ -1,3 +1,4 @@ +use std::cmp::Ordering; use std::fmt::{self, Debug}; use rustc_data_structures::captures::Captures; @@ -8,11 +9,14 @@ use rustc_index::bit_set::BitSet; use rustc_middle::mir::coverage::{CounterId, CovTerm, Expression, ExpressionId, Op}; use tracing::{debug, debug_span, instrument}; -use crate::coverage::graph::{BasicCoverageBlock, CoverageGraph, TraverseCoverageGraphWithLoops}; +use crate::coverage::graph::{BasicCoverageBlock, CoverageGraph, ReadyFirstTraversal}; + +#[cfg(test)] +mod tests; /// The coverage counter or counter expression associated with a particular /// BCB node or BCB edge. -#[derive(Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] enum BcbCounter { Counter { id: CounterId }, Expression { id: ExpressionId }, @@ -43,8 +47,9 @@ struct BcbExpression { rhs: BcbCounter, } -#[derive(Debug)] -pub(super) enum CounterIncrementSite { +/// Enum representing either a node or an edge in the coverage graph. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(super) enum Site { Node { bcb: BasicCoverageBlock }, Edge { from_bcb: BasicCoverageBlock, to_bcb: BasicCoverageBlock }, } @@ -54,16 +59,10 @@ pub(super) enum CounterIncrementSite { pub(super) struct CoverageCounters { /// List of places where a counter-increment statement should be injected /// into MIR, each with its corresponding counter ID. - counter_increment_sites: IndexVec<CounterId, CounterIncrementSite>, + counter_increment_sites: IndexVec<CounterId, Site>, /// Coverage counters/expressions that are associated with individual BCBs. node_counters: IndexVec<BasicCoverageBlock, Option<BcbCounter>>, - /// Coverage counters/expressions that are associated with the control-flow - /// edge between two BCBs. - /// - /// We currently don't iterate over this map, but if we do in the future, - /// switch it back to `FxIndexMap` to avoid query stability hazards. - edge_counters: FxHashMap<(BasicCoverageBlock, BasicCoverageBlock), BcbCounter>, /// Table of expression data, associating each expression ID with its /// corresponding operator (+ or -) and its LHS/RHS operands. @@ -83,93 +82,30 @@ impl CoverageCounters { let mut builder = CountersBuilder::new(graph, bcb_needs_counter); builder.make_bcb_counters(); - builder.counters + builder.into_coverage_counters() } fn with_num_bcbs(num_bcbs: usize) -> Self { Self { counter_increment_sites: IndexVec::new(), node_counters: IndexVec::from_elem_n(None, num_bcbs), - edge_counters: FxHashMap::default(), expressions: IndexVec::new(), expressions_memo: FxHashMap::default(), } } - /// Shared helper used by [`Self::make_phys_node_counter`] and - /// [`Self::make_phys_edge_counter`]. Don't call this directly. - fn make_counter_inner(&mut self, site: CounterIncrementSite) -> BcbCounter { + /// Creates a new physical counter for a BCB node or edge. + fn make_phys_counter(&mut self, site: Site) -> BcbCounter { let id = self.counter_increment_sites.push(site); BcbCounter::Counter { id } } - /// Creates a new physical counter for a BCB node. - fn make_phys_node_counter(&mut self, bcb: BasicCoverageBlock) -> BcbCounter { - self.make_counter_inner(CounterIncrementSite::Node { bcb }) - } - - /// Creates a new physical counter for a BCB edge. - fn make_phys_edge_counter( - &mut self, - from_bcb: BasicCoverageBlock, - to_bcb: BasicCoverageBlock, - ) -> BcbCounter { - self.make_counter_inner(CounterIncrementSite::Edge { from_bcb, to_bcb }) - } - fn make_expression(&mut self, lhs: BcbCounter, op: Op, rhs: BcbCounter) -> BcbCounter { let new_expr = BcbExpression { lhs, op, rhs }; - *self - .expressions_memo - .entry(new_expr) - .or_insert_with(|| Self::make_expression_inner(&mut self.expressions, new_expr)) - } - - /// This is an associated function so that we can call it while borrowing - /// `&mut self.expressions_memo`. - fn make_expression_inner( - expressions: &mut IndexVec<ExpressionId, BcbExpression>, - new_expr: BcbExpression, - ) -> BcbCounter { - // Simplify expressions using basic algebra. - // - // Some of these cases might not actually occur in practice, depending - // on the details of how the instrumentor builds expressions. - let BcbExpression { lhs, op, rhs } = new_expr; - - if let BcbCounter::Expression { id } = lhs { - let lhs_expr = &expressions[id]; - - // Simplify `(a - b) + b` to `a`. - if lhs_expr.op == Op::Subtract && op == Op::Add && lhs_expr.rhs == rhs { - return lhs_expr.lhs; - } - // Simplify `(a + b) - b` to `a`. - if lhs_expr.op == Op::Add && op == Op::Subtract && lhs_expr.rhs == rhs { - return lhs_expr.lhs; - } - // Simplify `(a + b) - a` to `b`. - if lhs_expr.op == Op::Add && op == Op::Subtract && lhs_expr.lhs == rhs { - return lhs_expr.rhs; - } - } - - if let BcbCounter::Expression { id } = rhs { - let rhs_expr = &expressions[id]; - - // Simplify `a + (b - a)` to `b`. - if op == Op::Add && rhs_expr.op == Op::Subtract && lhs == rhs_expr.rhs { - return rhs_expr.lhs; - } - // Simplify `a - (a - b)` to `b`. - if op == Op::Subtract && rhs_expr.op == Op::Subtract && lhs == rhs_expr.lhs { - return rhs_expr.rhs; - } - } - - // Simplification failed, so actually create the new expression. - let id = expressions.push(new_expr); - BcbCounter::Expression { id } + *self.expressions_memo.entry(new_expr).or_insert_with(|| { + let id = self.expressions.push(new_expr); + BcbCounter::Expression { id } + }) } /// Creates a counter that is the sum of the given counters. @@ -182,6 +118,12 @@ impl CoverageCounters { .reduce(|accum, counter| self.make_expression(accum, Op::Add, counter)) } + /// Creates a counter whose value is `lhs - SUM(rhs)`. + fn make_subtracted_sum(&mut self, lhs: BcbCounter, rhs: &[BcbCounter]) -> BcbCounter { + let Some(rhs_sum) = self.make_sum(rhs) else { return lhs }; + self.make_expression(lhs, Op::Subtract, rhs_sum) + } + pub(super) fn num_counters(&self) -> usize { self.counter_increment_sites.len() } @@ -195,20 +137,6 @@ impl CoverageCounters { counter } - fn set_edge_counter( - &mut self, - from_bcb: BasicCoverageBlock, - to_bcb: BasicCoverageBlock, - counter: BcbCounter, - ) -> BcbCounter { - let existing = self.edge_counters.insert((from_bcb, to_bcb), counter); - assert!( - existing.is_none(), - "edge ({from_bcb:?} -> {to_bcb:?}) already has a counter: {existing:?} => {counter:?}" - ); - counter - } - pub(super) fn term_for_bcb(&self, bcb: BasicCoverageBlock) -> Option<CovTerm> { self.node_counters[bcb].map(|counter| counter.as_term()) } @@ -218,8 +146,8 @@ impl CoverageCounters { /// each site's corresponding counter ID. pub(super) fn counter_increment_sites( &self, - ) -> impl Iterator<Item = (CounterId, &CounterIncrementSite)> { - self.counter_increment_sites.iter_enumerated() + ) -> impl Iterator<Item = (CounterId, Site)> + Captures<'_> { + self.counter_increment_sites.iter_enumerated().map(|(id, &site)| (id, site)) } /// Returns an iterator over the subset of BCB nodes that have been associated @@ -254,22 +182,53 @@ impl CoverageCounters { } } +/// Symbolic representation of the coverage counter to be used for a particular +/// node or edge in the coverage graph. The same site counter can be used for +/// multiple sites, if they have been determined to have the same count. +#[derive(Clone, Copy, Debug)] +enum SiteCounter { + /// A physical counter at some node/edge. + Phys { site: Site }, + /// A counter expression for a node that takes the sum of all its in-edge + /// counters. + NodeSumExpr { bcb: BasicCoverageBlock }, + /// A counter expression for an edge that takes the counter of its source + /// node, and subtracts the counters of all its sibling out-edges. + EdgeDiffExpr { from_bcb: BasicCoverageBlock, to_bcb: BasicCoverageBlock }, +} + +/// Yields the graph successors of `from_bcb` that aren't `to_bcb`. This is +/// used when creating a counter expression for [`SiteCounter::EdgeDiffExpr`]. +/// +/// For example, in this diagram the sibling out-edge targets of edge `AC` are +/// the nodes `B` and `D`. +/// +/// ```text +/// A +/// / | \ +/// B C D +/// ``` +fn sibling_out_edge_targets( + graph: &CoverageGraph, + from_bcb: BasicCoverageBlock, + to_bcb: BasicCoverageBlock, +) -> impl Iterator<Item = BasicCoverageBlock> + Captures<'_> { + graph.successors[from_bcb].iter().copied().filter(move |&t| t != to_bcb) +} + /// Helper struct that allows counter creation to inspect the BCB graph, and /// the set of nodes that need counters. struct CountersBuilder<'a> { - counters: CoverageCounters, graph: &'a CoverageGraph, bcb_needs_counter: &'a BitSet<BasicCoverageBlock>, + + site_counters: FxHashMap<Site, SiteCounter>, } impl<'a> CountersBuilder<'a> { fn new(graph: &'a CoverageGraph, bcb_needs_counter: &'a BitSet<BasicCoverageBlock>) -> Self { assert_eq!(graph.num_nodes(), bcb_needs_counter.domain_size()); - Self { - counters: CoverageCounters::with_num_bcbs(graph.num_nodes()), - graph, - bcb_needs_counter, - } + Self { graph, bcb_needs_counter, site_counters: FxHashMap::default() } } fn make_bcb_counters(&mut self) { @@ -277,23 +236,12 @@ impl<'a> CountersBuilder<'a> { // Traverse the coverage graph, ensuring that every node that needs a // coverage counter has one. - // - // The traversal tries to ensure that, when a loop is encountered, all - // nodes within the loop are visited before visiting any nodes outside - // the loop. - let mut traversal = TraverseCoverageGraphWithLoops::new(self.graph); - while let Some(bcb) = traversal.next() { + for bcb in ReadyFirstTraversal::new(self.graph) { let _span = debug_span!("traversal", ?bcb).entered(); if self.bcb_needs_counter.contains(bcb) { self.make_node_counter_and_out_edge_counters(bcb); } } - - assert!( - traversal.is_complete(), - "`TraverseCoverageGraphWithLoops` missed some `BasicCoverageBlock`s: {:?}", - traversal.unvisited(), - ); } /// Make sure the given node has a node counter, and then make sure each of @@ -302,9 +250,7 @@ impl<'a> CountersBuilder<'a> { fn make_node_counter_and_out_edge_counters(&mut self, from_bcb: BasicCoverageBlock) { // First, ensure that this node has a counter of some kind. // We might also use that counter to compute one of the out-edge counters. - let node_counter = self.get_or_make_node_counter(from_bcb); - - let successors = self.graph.successors[from_bcb].as_slice(); + self.get_or_make_node_counter(from_bcb); // If this node's out-edges won't sum to the node's counter, // then there's no reason to create edge counters here. @@ -315,11 +261,11 @@ impl<'a> CountersBuilder<'a> { // When choosing which out-edge should be given a counter expression, ignore edges that // already have counters, or could use the existing counter of their target node. let out_edge_has_counter = |to_bcb| { - if self.counters.edge_counters.contains_key(&(from_bcb, to_bcb)) { + if self.site_counters.contains_key(&Site::Edge { from_bcb, to_bcb }) { return true; } self.graph.sole_predecessor(to_bcb) == Some(from_bcb) - && self.counters.node_counters[to_bcb].is_some() + && self.site_counters.contains_key(&Site::Node { bcb: to_bcb }) }; // Determine the set of out-edges that could benefit from being given an expression. @@ -332,51 +278,41 @@ impl<'a> CountersBuilder<'a> { // If there are out-edges without counters, choose one to be given an expression // (computed from this node and the other out-edges) instead of a physical counter. - let Some(target_bcb) = self.choose_out_edge_for_expression(from_bcb, &candidate_successors) + let Some(to_bcb) = self.choose_out_edge_for_expression(from_bcb, &candidate_successors) else { return; }; // For each out-edge other than the one that was chosen to get an expression, - // ensure that it has a counter (existing counter/expression or a new counter), - // and accumulate the corresponding counters into a single sum expression. - let other_out_edge_counters = successors - .iter() - .copied() - // Skip the chosen edge, since we'll calculate its count from this sum. - .filter(|&edge_target_bcb| edge_target_bcb != target_bcb) - .map(|to_bcb| self.get_or_make_edge_counter(from_bcb, to_bcb)) - .collect::<Vec<_>>(); - let Some(sum_of_all_other_out_edges) = self.counters.make_sum(&other_out_edge_counters) - else { - return; - }; + // ensure that it has a counter (existing counter/expression or a new counter). + for target in sibling_out_edge_targets(self.graph, from_bcb, to_bcb) { + self.get_or_make_edge_counter(from_bcb, target); + } // Now create an expression for the chosen edge, by taking the counter // for its source node and subtracting the sum of its sibling out-edges. - let expression = - self.counters.make_expression(node_counter, Op::Subtract, sum_of_all_other_out_edges); - - debug!("{target_bcb:?} gets an expression: {expression:?}"); - self.counters.set_edge_counter(from_bcb, target_bcb, expression); + let counter = SiteCounter::EdgeDiffExpr { from_bcb, to_bcb }; + self.site_counters.insert(Site::Edge { from_bcb, to_bcb }, counter); } #[instrument(level = "debug", skip(self))] - fn get_or_make_node_counter(&mut self, bcb: BasicCoverageBlock) -> BcbCounter { + fn get_or_make_node_counter(&mut self, bcb: BasicCoverageBlock) -> SiteCounter { // If the BCB already has a counter, return it. - if let Some(counter) = self.counters.node_counters[bcb] { + if let Some(&counter) = self.site_counters.get(&Site::Node { bcb }) { debug!("{bcb:?} already has a counter: {counter:?}"); return counter; } let counter = self.make_node_counter_inner(bcb); - self.counters.set_node_counter(bcb, counter) + self.site_counters.insert(Site::Node { bcb }, counter); + counter } - fn make_node_counter_inner(&mut self, bcb: BasicCoverageBlock) -> BcbCounter { + fn make_node_counter_inner(&mut self, bcb: BasicCoverageBlock) -> SiteCounter { // If the node's sole in-edge already has a counter, use that. if let Some(sole_pred) = self.graph.sole_predecessor(bcb) - && let Some(&edge_counter) = self.counters.edge_counters.get(&(sole_pred, bcb)) + && let Some(&edge_counter) = + self.site_counters.get(&Site::Edge { from_bcb: sole_pred, to_bcb: bcb }) { return edge_counter; } @@ -390,20 +326,17 @@ impl<'a> CountersBuilder<'a> { // leading to infinite recursion. if predecessors.len() <= 1 || predecessors.contains(&bcb) { debug!(?bcb, ?predecessors, "node has <=1 predecessors or is its own predecessor"); - let counter = self.counters.make_phys_node_counter(bcb); + let counter = SiteCounter::Phys { site: Site::Node { bcb } }; debug!(?bcb, ?counter, "node gets a physical counter"); return counter; } // A BCB with multiple incoming edges can compute its count by ensuring that counters // exist for each of those edges, and then adding them up to get a total count. - let in_edge_counters = predecessors - .iter() - .copied() - .map(|from_bcb| self.get_or_make_edge_counter(from_bcb, bcb)) - .collect::<Vec<_>>(); - let sum_of_in_edges: BcbCounter = - self.counters.make_sum(&in_edge_counters).expect("there must be at least one in-edge"); + for &from_bcb in predecessors { + self.get_or_make_edge_counter(from_bcb, bcb); + } + let sum_of_in_edges = SiteCounter::NodeSumExpr { bcb }; debug!("{bcb:?} gets a new counter (sum of predecessor counters): {sum_of_in_edges:?}"); sum_of_in_edges @@ -414,22 +347,23 @@ impl<'a> CountersBuilder<'a> { &mut self, from_bcb: BasicCoverageBlock, to_bcb: BasicCoverageBlock, - ) -> BcbCounter { + ) -> SiteCounter { // If the edge already has a counter, return it. - if let Some(&counter) = self.counters.edge_counters.get(&(from_bcb, to_bcb)) { + if let Some(&counter) = self.site_counters.get(&Site::Edge { from_bcb, to_bcb }) { debug!("Edge {from_bcb:?}->{to_bcb:?} already has a counter: {counter:?}"); return counter; } let counter = self.make_edge_counter_inner(from_bcb, to_bcb); - self.counters.set_edge_counter(from_bcb, to_bcb, counter) + self.site_counters.insert(Site::Edge { from_bcb, to_bcb }, counter); + counter } fn make_edge_counter_inner( &mut self, from_bcb: BasicCoverageBlock, to_bcb: BasicCoverageBlock, - ) -> BcbCounter { + ) -> SiteCounter { // If the target node has exactly one in-edge (i.e. this one), then just // use the node's counter, since it will have the same value. if let Some(sole_pred) = self.graph.sole_predecessor(to_bcb) { @@ -447,7 +381,7 @@ impl<'a> CountersBuilder<'a> { } // Make a new counter to count this edge. - let counter = self.counters.make_phys_edge_counter(from_bcb, to_bcb); + let counter = SiteCounter::Phys { site: Site::Edge { from_bcb, to_bcb } }; debug!(?from_bcb, ?to_bcb, ?counter, "edge gets a physical counter"); counter } @@ -510,4 +444,145 @@ impl<'a> CountersBuilder<'a> { None } + + fn into_coverage_counters(self) -> CoverageCounters { + Transcriber::new(&self).transcribe_counters() + } +} + +/// Helper struct for converting `CountersBuilder` into a final `CoverageCounters`. +struct Transcriber<'a> { + old: &'a CountersBuilder<'a>, + new: CoverageCounters, + phys_counter_for_site: FxHashMap<Site, BcbCounter>, +} + +impl<'a> Transcriber<'a> { + fn new(old: &'a CountersBuilder<'a>) -> Self { + Self { + old, + new: CoverageCounters::with_num_bcbs(old.graph.num_nodes()), + phys_counter_for_site: FxHashMap::default(), + } + } + + fn transcribe_counters(mut self) -> CoverageCounters { + for bcb in self.old.bcb_needs_counter.iter() { + let site = Site::Node { bcb }; + let site_counter = self.site_counter(site); + + // Resolve the site counter into flat lists of nodes/edges whose + // physical counts contribute to the counter for this node. + // Distinguish between counts that will be added vs subtracted. + let mut pos = vec![]; + let mut neg = vec![]; + self.push_resolved_sites(site_counter, &mut pos, &mut neg); + + // Simplify by cancelling out sites that appear on both sides. + let (mut pos, mut neg) = sort_and_cancel(pos, neg); + + if pos.is_empty() { + // If we somehow end up with no positive terms after cancellation, + // fall back to creating a physical counter. There's no known way + // for this to happen, but it's hard to confidently rule it out. + debug_assert!(false, "{site:?} has no positive counter terms"); + pos = vec![Some(site)]; + neg = vec![]; + } + + let mut new_counters_for_sites = |sites: Vec<Option<Site>>| { + sites + .into_iter() + .filter_map(|id| try { self.ensure_phys_counter(id?) }) + .collect::<Vec<_>>() + }; + let mut pos = new_counters_for_sites(pos); + let mut neg = new_counters_for_sites(neg); + + pos.sort(); + neg.sort(); + + let pos_counter = self.new.make_sum(&pos).expect("`pos` should not be empty"); + let new_counter = self.new.make_subtracted_sum(pos_counter, &neg); + self.new.set_node_counter(bcb, new_counter); + } + + self.new + } + + fn site_counter(&self, site: Site) -> SiteCounter { + self.old.site_counters.get(&site).copied().unwrap_or_else(|| { + // We should have already created all necessary site counters. + // But if we somehow didn't, avoid crashing in release builds, + // and just use an extra physical counter instead. + debug_assert!(false, "{site:?} should have a counter"); + SiteCounter::Phys { site } + }) + } + + fn ensure_phys_counter(&mut self, site: Site) -> BcbCounter { + *self.phys_counter_for_site.entry(site).or_insert_with(|| self.new.make_phys_counter(site)) + } + + /// Resolves the given counter into flat lists of nodes/edges, whose counters + /// will then be added and subtracted to form a counter expression. + fn push_resolved_sites(&self, counter: SiteCounter, pos: &mut Vec<Site>, neg: &mut Vec<Site>) { + match counter { + SiteCounter::Phys { site } => pos.push(site), + SiteCounter::NodeSumExpr { bcb } => { + for &from_bcb in &self.old.graph.predecessors[bcb] { + let edge_counter = self.site_counter(Site::Edge { from_bcb, to_bcb: bcb }); + self.push_resolved_sites(edge_counter, pos, neg); + } + } + SiteCounter::EdgeDiffExpr { from_bcb, to_bcb } => { + // First, add the count for `from_bcb`. + let node_counter = self.site_counter(Site::Node { bcb: from_bcb }); + self.push_resolved_sites(node_counter, pos, neg); + + // Then subtract the counts for the other out-edges. + for target in sibling_out_edge_targets(self.old.graph, from_bcb, to_bcb) { + let edge_counter = self.site_counter(Site::Edge { from_bcb, to_bcb: target }); + // Swap `neg` and `pos` so that the counter is subtracted. + self.push_resolved_sites(edge_counter, neg, pos); + } + } + } + } +} + +/// Given two lists: +/// - Sorts each list. +/// - Converts each list to `Vec<Option<T>>`. +/// - Scans for values that appear in both lists, and cancels them out by +/// replacing matching pairs of values with `None`. +fn sort_and_cancel<T: Ord>(mut pos: Vec<T>, mut neg: Vec<T>) -> (Vec<Option<T>>, Vec<Option<T>>) { + pos.sort(); + neg.sort(); + + // Convert to `Vec<Option<T>>`. If `T` has a niche, this should be zero-cost. + let mut pos = pos.into_iter().map(Some).collect::<Vec<_>>(); + let mut neg = neg.into_iter().map(Some).collect::<Vec<_>>(); + + // Scan through the lists using two cursors. When either cursor reaches the + // end of its list, there can be no more equal pairs, so stop. + let mut p = 0; + let mut n = 0; + while p < pos.len() && n < neg.len() { + // If the values are equal, remove them and advance both cursors. + // Otherwise, advance whichever cursor points to the lesser value. + // (Choosing which cursor to advance relies on both lists being sorted.) + match pos[p].cmp(&neg[n]) { + Ordering::Less => p += 1, + Ordering::Equal => { + pos[p] = None; + neg[n] = None; + p += 1; + n += 1; + } + Ordering::Greater => n += 1, + } + } + + (pos, neg) } diff --git a/compiler/rustc_mir_transform/src/coverage/counters/tests.rs b/compiler/rustc_mir_transform/src/coverage/counters/tests.rs new file mode 100644 index 00000000000..794d4358f82 --- /dev/null +++ b/compiler/rustc_mir_transform/src/coverage/counters/tests.rs @@ -0,0 +1,41 @@ +use std::fmt::Debug; + +use super::sort_and_cancel; + +fn flatten<T>(input: Vec<Option<T>>) -> Vec<T> { + input.into_iter().flatten().collect() +} + +fn sort_and_cancel_and_flatten<T: Clone + Ord>(pos: Vec<T>, neg: Vec<T>) -> (Vec<T>, Vec<T>) { + let (pos_actual, neg_actual) = sort_and_cancel(pos, neg); + (flatten(pos_actual), flatten(neg_actual)) +} + +#[track_caller] +fn check_test_case<T: Clone + Debug + Ord>( + pos: Vec<T>, + neg: Vec<T>, + pos_expected: Vec<T>, + neg_expected: Vec<T>, +) { + eprintln!("pos = {pos:?}; neg = {neg:?}"); + let output = sort_and_cancel_and_flatten(pos, neg); + assert_eq!(output, (pos_expected, neg_expected)); +} + +#[test] +fn cancellation() { + let cases: &[(Vec<u32>, Vec<u32>, Vec<u32>, Vec<u32>)] = &[ + (vec![], vec![], vec![], vec![]), + (vec![4, 2, 1, 5, 3], vec![], vec![1, 2, 3, 4, 5], vec![]), + (vec![5, 5, 5, 5, 5], vec![5], vec![5, 5, 5, 5], vec![]), + (vec![1, 1, 2, 2, 3, 3], vec![1, 2, 3], vec![1, 2, 3], vec![]), + (vec![1, 1, 2, 2, 3, 3], vec![2, 4, 2], vec![1, 1, 3, 3], vec![4]), + ]; + + for (pos, neg, pos_expected, neg_expected) in cases { + check_test_case(pos.to_vec(), neg.to_vec(), pos_expected.to_vec(), neg_expected.to_vec()); + // Same test case, but with its inputs flipped and its outputs flipped. + check_test_case(neg.to_vec(), pos.to_vec(), neg_expected.to_vec(), pos_expected.to_vec()); + } +} diff --git a/compiler/rustc_mir_transform/src/coverage/graph.rs b/compiler/rustc_mir_transform/src/coverage/graph.rs index 168262bf01c..ad6774fccd6 100644 --- a/compiler/rustc_mir_transform/src/coverage/graph.rs +++ b/compiler/rustc_mir_transform/src/coverage/graph.rs @@ -1,7 +1,7 @@ use std::cmp::Ordering; use std::collections::VecDeque; -use std::iter; use std::ops::{Index, IndexMut}; +use std::{iter, mem, slice}; use rustc_data_structures::captures::Captures; use rustc_data_structures::fx::FxHashSet; @@ -9,7 +9,6 @@ use rustc_data_structures::graph::dominators::Dominators; use rustc_data_structures::graph::{self, DirectedGraph, StartNode}; use rustc_index::IndexVec; use rustc_index::bit_set::BitSet; -use rustc_middle::bug; use rustc_middle::mir::{self, BasicBlock, Terminator, TerminatorKind}; use tracing::debug; @@ -127,10 +126,10 @@ impl CoverageGraph { let mut bcbs = IndexVec::<BasicCoverageBlock, _>::with_capacity(num_basic_blocks); let mut bb_to_bcb = IndexVec::from_elem_n(None, num_basic_blocks); - let mut add_basic_coverage_block = |basic_blocks: &mut Vec<BasicBlock>| { + let mut flush_chain_into_new_bcb = |current_chain: &mut Vec<BasicBlock>| { // Take the accumulated list of blocks, leaving the vector empty // to be used by subsequent BCBs. - let basic_blocks = std::mem::take(basic_blocks); + let basic_blocks = mem::take(current_chain); let bcb = bcbs.next_index(); for &bb in basic_blocks.iter() { @@ -141,48 +140,41 @@ impl CoverageGraph { bcb_filtered_successors(mir_body[bb].terminator()).is_out_summable() }); let bcb_data = BasicCoverageBlockData { basic_blocks, is_out_summable }; - debug!("adding bcb{}: {:?}", bcb.index(), bcb_data); + debug!("adding {bcb:?}: {bcb_data:?}"); bcbs.push(bcb_data); }; - // Walk the MIR CFG using a Preorder traversal, which starts from `START_BLOCK` and follows - // each block terminator's `successors()`. Coverage spans must map to actual source code, - // so compiler generated blocks and paths can be ignored. To that end, the CFG traversal - // intentionally omits unwind paths. - // FIXME(#78544): MIR InstrumentCoverage: Improve coverage of `#[should_panic]` tests and - // `catch_unwind()` handlers. + // Traverse the MIR control-flow graph, accumulating chains of blocks + // that can be combined into a single node in the coverage graph. + // A depth-first search ensures that if two nodes can be chained + // together, they will be adjacent in the traversal order. // Accumulates a chain of blocks that will be combined into one BCB. - let mut basic_blocks = Vec::new(); + let mut current_chain = vec![]; - let filtered_successors = |bb| bcb_filtered_successors(mir_body[bb].terminator()); - for bb in short_circuit_preorder(mir_body, filtered_successors) + let subgraph = CoverageRelevantSubgraph::new(&mir_body.basic_blocks); + for bb in graph::depth_first_search(subgraph, mir::START_BLOCK) .filter(|&bb| mir_body[bb].terminator().kind != TerminatorKind::Unreachable) { - // If the previous block can't be chained into `bb`, flush the accumulated - // blocks into a new BCB, then start building the next chain. - if let Some(&prev) = basic_blocks.last() - && (!filtered_successors(prev).is_chainable() || { - // If `bb` has multiple predecessor blocks, or `prev` isn't - // one of its predecessors, we can't chain and must flush. - let predecessors = &mir_body.basic_blocks.predecessors()[bb]; - predecessors.len() > 1 || !predecessors.contains(&prev) - }) - { - debug!( - terminator_kind = ?mir_body[prev].terminator().kind, - predecessors = ?&mir_body.basic_blocks.predecessors()[bb], - "can't chain from {prev:?} to {bb:?}" - ); - add_basic_coverage_block(&mut basic_blocks); + if let Some(&prev) = current_chain.last() { + // Adding a block to a non-empty chain is allowed if the + // previous block permits chaining, and the current block has + // `prev` as its sole predecessor. + let can_chain = subgraph.coverage_successors(prev).is_out_chainable() + && mir_body.basic_blocks.predecessors()[bb].as_slice() == &[prev]; + if !can_chain { + // The current block can't be added to the existing chain, so + // flush that chain into a new BCB, and start a new chain. + flush_chain_into_new_bcb(&mut current_chain); + } } - basic_blocks.push(bb); + current_chain.push(bb); } - if !basic_blocks.is_empty() { + if !current_chain.is_empty() { debug!("flushing accumulated blocks into one last BCB"); - add_basic_coverage_block(&mut basic_blocks); + flush_chain_into_new_bcb(&mut current_chain); } (bcbs, bb_to_bcb) @@ -389,34 +381,28 @@ impl BasicCoverageBlockData { /// indicates whether that block can potentially be combined into the same BCB /// as its sole successor. #[derive(Clone, Copy, Debug)] -enum CoverageSuccessors<'a> { - /// The terminator has exactly one straight-line successor, so its block can - /// potentially be combined into the same BCB as that successor. - Chainable(BasicBlock), - /// The block cannot be combined into the same BCB as its successor(s). - NotChainable(&'a [BasicBlock]), - /// Yield terminators are not chainable, and their execution count can also - /// differ from the execution count of their out-edge. - Yield(BasicBlock), +struct CoverageSuccessors<'a> { + /// Coverage-relevant successors of the corresponding terminator. + /// There might be 0, 1, or multiple targets. + targets: &'a [BasicBlock], + /// `Yield` terminators are not chainable, because their sole out-edge is + /// only followed if/when the generator is resumed after the yield. + is_yield: bool, } impl CoverageSuccessors<'_> { - fn is_chainable(&self) -> bool { - match self { - Self::Chainable(_) => true, - Self::NotChainable(_) => false, - Self::Yield(_) => false, - } + /// If `false`, this terminator cannot be chained into another block when + /// building the coverage graph. + fn is_out_chainable(&self) -> bool { + // If a terminator is out-summable and has exactly one out-edge, then + // it is eligible to be chained into its successor block. + self.is_out_summable() && self.targets.len() == 1 } /// Returns true if the terminator itself is assumed to have the same /// execution count as the sum of its out-edges (assuming no panics). fn is_out_summable(&self) -> bool { - match self { - Self::Chainable(_) => true, - Self::NotChainable(_) => true, - Self::Yield(_) => false, - } + !self.is_yield && !self.targets.is_empty() } } @@ -425,12 +411,7 @@ impl IntoIterator for CoverageSuccessors<'_> { type IntoIter = impl DoubleEndedIterator<Item = Self::Item>; fn into_iter(self) -> Self::IntoIter { - match self { - Self::Chainable(bb) | Self::Yield(bb) => { - Some(bb).into_iter().chain((&[]).iter().copied()) - } - Self::NotChainable(bbs) => None.into_iter().chain(bbs.iter().copied()), - } + self.targets.iter().copied() } } @@ -440,14 +421,17 @@ impl IntoIterator for CoverageSuccessors<'_> { // `catch_unwind()` handlers. fn bcb_filtered_successors<'a, 'tcx>(terminator: &'a Terminator<'tcx>) -> CoverageSuccessors<'a> { use TerminatorKind::*; - match terminator.kind { + let mut is_yield = false; + let targets = match &terminator.kind { // A switch terminator can have many coverage-relevant successors. - // (If there is exactly one successor, we still treat it as not chainable.) - SwitchInt { ref targets, .. } => CoverageSuccessors::NotChainable(targets.all_targets()), + SwitchInt { targets, .. } => targets.all_targets(), // A yield terminator has exactly 1 successor, but should not be chained, // because its resume edge has a different execution count. - Yield { resume, .. } => CoverageSuccessors::Yield(resume), + Yield { resume, .. } => { + is_yield = true; + slice::from_ref(resume) + } // These terminators have exactly one coverage-relevant successor, // and can be chained into it. @@ -455,24 +439,15 @@ fn bcb_filtered_successors<'a, 'tcx>(terminator: &'a Terminator<'tcx>) -> Covera | Drop { target, .. } | FalseEdge { real_target: target, .. } | FalseUnwind { real_target: target, .. } - | Goto { target } => CoverageSuccessors::Chainable(target), + | Goto { target } => slice::from_ref(target), // A call terminator can normally be chained, except when it has no // successor because it is known to diverge. - Call { target: maybe_target, .. } => match maybe_target { - Some(target) => CoverageSuccessors::Chainable(target), - None => CoverageSuccessors::NotChainable(&[]), - }, + Call { target: maybe_target, .. } => maybe_target.as_slice(), // An inline asm terminator can normally be chained, except when it // diverges or uses asm goto. - InlineAsm { ref targets, .. } => { - if let [target] = targets[..] { - CoverageSuccessors::Chainable(target) - } else { - CoverageSuccessors::NotChainable(targets) - } - } + InlineAsm { targets, .. } => &targets, // These terminators have no coverage-relevant successors. CoroutineDrop @@ -480,164 +455,160 @@ fn bcb_filtered_successors<'a, 'tcx>(terminator: &'a Terminator<'tcx>) -> Covera | TailCall { .. } | Unreachable | UnwindResume - | UnwindTerminate(_) => CoverageSuccessors::NotChainable(&[]), - } + | UnwindTerminate(_) => &[], + }; + + CoverageSuccessors { targets, is_yield } } -/// Maintains separate worklists for each loop in the BasicCoverageBlock CFG, plus one for the -/// CoverageGraph outside all loops. This supports traversing the BCB CFG in a way that -/// ensures a loop is completely traversed before processing Blocks after the end of the loop. -#[derive(Debug)] -struct TraversalContext { - /// BCB with one or more incoming loop backedges, indicating which loop - /// this context is for. - /// - /// If `None`, this is the non-loop context for the function as a whole. - loop_header: Option<BasicCoverageBlock>, +/// Wrapper around a [`mir::BasicBlocks`] graph that restricts each node's +/// successors to only the ones considered "relevant" when building a coverage +/// graph. +#[derive(Clone, Copy)] +struct CoverageRelevantSubgraph<'a, 'tcx> { + basic_blocks: &'a mir::BasicBlocks<'tcx>, +} +impl<'a, 'tcx> CoverageRelevantSubgraph<'a, 'tcx> { + fn new(basic_blocks: &'a mir::BasicBlocks<'tcx>) -> Self { + Self { basic_blocks } + } - /// Worklist of BCBs to be processed in this context. - worklist: VecDeque<BasicCoverageBlock>, + fn coverage_successors(&self, bb: BasicBlock) -> CoverageSuccessors<'_> { + bcb_filtered_successors(self.basic_blocks[bb].terminator()) + } } +impl<'a, 'tcx> graph::DirectedGraph for CoverageRelevantSubgraph<'a, 'tcx> { + type Node = BasicBlock; -pub(crate) struct TraverseCoverageGraphWithLoops<'a> { - basic_coverage_blocks: &'a CoverageGraph, + fn num_nodes(&self) -> usize { + self.basic_blocks.num_nodes() + } +} +impl<'a, 'tcx> graph::Successors for CoverageRelevantSubgraph<'a, 'tcx> { + fn successors(&self, bb: Self::Node) -> impl Iterator<Item = Self::Node> { + self.coverage_successors(bb).into_iter() + } +} - context_stack: Vec<TraversalContext>, - visited: BitSet<BasicCoverageBlock>, +/// State of a node in the coverage graph during ready-first traversal. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +enum ReadyState { + /// This node has not yet been added to the fallback queue or ready queue. + Unqueued, + /// This node is currently in the fallback queue. + InFallbackQueue, + /// This node's predecessors have all been visited, so it is in the ready queue. + /// (It might also have a stale entry in the fallback queue.) + InReadyQueue, + /// This node has been visited. + /// (It might also have a stale entry in the fallback queue.) + Visited, } -impl<'a> TraverseCoverageGraphWithLoops<'a> { - pub(crate) fn new(basic_coverage_blocks: &'a CoverageGraph) -> Self { - let worklist = VecDeque::from([basic_coverage_blocks.start_node()]); - let context_stack = vec![TraversalContext { loop_header: None, worklist }]; - - // `context_stack` starts with a `TraversalContext` for the main function context (beginning - // with the `start` BasicCoverageBlock of the function). New worklists are pushed to the top - // of the stack as loops are entered, and popped off of the stack when a loop's worklist is - // exhausted. - let visited = BitSet::new_empty(basic_coverage_blocks.num_nodes()); - Self { basic_coverage_blocks, context_stack, visited } - } +/// Iterator that visits nodes in the coverage graph, in an order that always +/// prefers "ready" nodes whose predecessors have already been visited. +pub(crate) struct ReadyFirstTraversal<'a> { + graph: &'a CoverageGraph, + + /// For each node, the number of its predecessor nodes that haven't been visited yet. + n_unvisited_preds: IndexVec<BasicCoverageBlock, u32>, + /// Indicates whether a node has been visited, or which queue it is in. + state: IndexVec<BasicCoverageBlock, ReadyState>, + + /// Holds unvisited nodes whose predecessors have all been visited. + ready_queue: VecDeque<BasicCoverageBlock>, + /// Holds unvisited nodes with some unvisited predecessors. + /// Also contains stale entries for nodes that were upgraded to ready. + fallback_queue: VecDeque<BasicCoverageBlock>, +} + +impl<'a> ReadyFirstTraversal<'a> { + pub(crate) fn new(graph: &'a CoverageGraph) -> Self { + let num_nodes = graph.num_nodes(); + + let n_unvisited_preds = + IndexVec::from_fn_n(|node| graph.predecessors[node].len() as u32, num_nodes); + let mut state = IndexVec::from_elem_n(ReadyState::Unqueued, num_nodes); - pub(crate) fn next(&mut self) -> Option<BasicCoverageBlock> { - debug!( - "TraverseCoverageGraphWithLoops::next - context_stack: {:?}", - self.context_stack.iter().rev().collect::<Vec<_>>() + // We know from coverage graph construction that the start node is the + // only node with no predecessors. + debug_assert!( + n_unvisited_preds.iter_enumerated().all(|(node, &n)| (node == START_BCB) == (n == 0)) ); + let ready_queue = VecDeque::from(vec![START_BCB]); + state[START_BCB] = ReadyState::InReadyQueue; - while let Some(context) = self.context_stack.last_mut() { - let Some(bcb) = context.worklist.pop_front() else { - // This stack level is exhausted; pop it and try the next one. - self.context_stack.pop(); - continue; - }; + Self { graph, state, n_unvisited_preds, ready_queue, fallback_queue: VecDeque::new() } + } - if !self.visited.insert(bcb) { - debug!("Already visited: {bcb:?}"); - continue; - } - debug!("Visiting {bcb:?}"); + /// Returns the next node from the ready queue, or else the next unvisited + /// node from the fallback queue. + fn next_inner(&mut self) -> Option<BasicCoverageBlock> { + // Always prefer to yield a ready node if possible. + if let Some(node) = self.ready_queue.pop_front() { + assert_eq!(self.state[node], ReadyState::InReadyQueue); + return Some(node); + } - if self.basic_coverage_blocks.is_loop_header.contains(bcb) { - debug!("{bcb:?} is a loop header! Start a new TraversalContext..."); - self.context_stack - .push(TraversalContext { loop_header: Some(bcb), worklist: VecDeque::new() }); + while let Some(node) = self.fallback_queue.pop_front() { + match self.state[node] { + // This entry in the fallback queue is not stale, so yield it. + ReadyState::InFallbackQueue => return Some(node), + // This node was added to the fallback queue, but later became + // ready and was visited via the ready queue. Ignore it here. + ReadyState::Visited => {} + // Unqueued nodes can't be in the fallback queue, by definition. + // We know that the ready queue is empty at this point. + ReadyState::Unqueued | ReadyState::InReadyQueue => unreachable!( + "unexpected state for {node:?} in the fallback queue: {:?}", + self.state[node] + ), } - self.add_successors_to_worklists(bcb); - return Some(bcb); } None } - fn add_successors_to_worklists(&mut self, bcb: BasicCoverageBlock) { - let successors = &self.basic_coverage_blocks.successors[bcb]; - debug!("{:?} has {} successors:", bcb, successors.len()); - - for &successor in successors { - if successor == bcb { - debug!( - "{:?} has itself as its own successor. (Note, the compiled code will \ - generate an infinite loop.)", - bcb - ); - // Don't re-add this successor to the worklist. We are already processing it. - // FIXME: This claims to skip just the self-successor, but it actually skips - // all other successors as well. Does that matter? - break; - } + fn mark_visited_and_enqueue_successors(&mut self, node: BasicCoverageBlock) { + assert!(self.state[node] < ReadyState::Visited); + self.state[node] = ReadyState::Visited; + + // For each of this node's successors, decrease the successor's + // "unvisited predecessors" count, and enqueue it if appropriate. + for &succ in &self.graph.successors[node] { + let is_unqueued = match self.state[succ] { + ReadyState::Unqueued => true, + ReadyState::InFallbackQueue => false, + ReadyState::InReadyQueue => { + unreachable!("nodes in the ready queue have no unvisited predecessors") + } + // The successor was already visited via one of its other predecessors. + ReadyState::Visited => continue, + }; - // Add successors of the current BCB to the appropriate context. Successors that - // stay within a loop are added to the BCBs context worklist. Successors that - // exit the loop (they are not dominated by the loop header) must be reachable - // from other BCBs outside the loop, and they will be added to a different - // worklist. - // - // Branching blocks (with more than one successor) must be processed before - // blocks with only one successor, to prevent unnecessarily complicating - // `Expression`s by creating a Counter in a `BasicCoverageBlock` that the - // branching block would have given an `Expression` (or vice versa). - - let context = self - .context_stack - .iter_mut() - .rev() - .find(|context| match context.loop_header { - Some(loop_header) => { - self.basic_coverage_blocks.dominates(loop_header, successor) - } - None => true, - }) - .unwrap_or_else(|| bug!("should always fall back to the root non-loop context")); - debug!("adding to worklist for {:?}", context.loop_header); - - // FIXME: The code below had debug messages claiming to add items to a - // particular end of the worklist, but was confused about which end was - // which. The existing behaviour has been preserved for now, but it's - // unclear what the intended behaviour was. - - if self.basic_coverage_blocks.successors[successor].len() > 1 { - context.worklist.push_back(successor); - } else { - context.worklist.push_front(successor); + self.n_unvisited_preds[succ] -= 1; + if self.n_unvisited_preds[succ] == 0 { + // This node's predecessors have all been visited, so add it to + // the ready queue. If it's already in the fallback queue, that + // fallback entry will be ignored later. + self.state[succ] = ReadyState::InReadyQueue; + self.ready_queue.push_back(succ); + } else if is_unqueued { + // This node has unvisited predecessors, so add it to the + // fallback queue in case we run out of ready nodes later. + self.state[succ] = ReadyState::InFallbackQueue; + self.fallback_queue.push_back(succ); } } } - - pub(crate) fn is_complete(&self) -> bool { - self.visited.count() == self.visited.domain_size() - } - - pub(crate) fn unvisited(&self) -> Vec<BasicCoverageBlock> { - let mut unvisited_set: BitSet<BasicCoverageBlock> = - BitSet::new_filled(self.visited.domain_size()); - unvisited_set.subtract(&self.visited); - unvisited_set.iter().collect::<Vec<_>>() - } } -fn short_circuit_preorder<'a, 'tcx, F, Iter>( - body: &'a mir::Body<'tcx>, - filtered_successors: F, -) -> impl Iterator<Item = BasicBlock> + Captures<'a> + Captures<'tcx> -where - F: Fn(BasicBlock) -> Iter, - Iter: IntoIterator<Item = BasicBlock>, -{ - let mut visited = BitSet::new_empty(body.basic_blocks.len()); - let mut worklist = vec![mir::START_BLOCK]; - - std::iter::from_fn(move || { - while let Some(bb) = worklist.pop() { - if !visited.insert(bb) { - continue; - } - - worklist.extend(filtered_successors(bb)); +impl<'a> Iterator for ReadyFirstTraversal<'a> { + type Item = BasicCoverageBlock; - return Some(bb); - } - - None - }) + fn next(&mut self) -> Option<Self::Item> { + let node = self.next_inner()?; + self.mark_visited_and_enqueue_successors(node); + Some(node) + } } diff --git a/compiler/rustc_mir_transform/src/coverage/mappings.rs b/compiler/rustc_mir_transform/src/coverage/mappings.rs index 2db7c6cf1d6..4185b3f4d4d 100644 --- a/compiler/rustc_mir_transform/src/coverage/mappings.rs +++ b/compiler/rustc_mir_transform/src/coverage/mappings.rs @@ -80,7 +80,7 @@ pub(super) fn extract_all_mapping_info_from_mir<'tcx>( tcx: TyCtxt<'tcx>, mir_body: &mir::Body<'tcx>, hir_info: &ExtractedHirInfo, - basic_coverage_blocks: &CoverageGraph, + graph: &CoverageGraph, ) -> ExtractedMappings { let mut code_mappings = vec![]; let mut branch_pairs = vec![]; @@ -102,23 +102,23 @@ pub(super) fn extract_all_mapping_info_from_mir<'tcx>( } } else { // Extract coverage spans from MIR statements/terminators as normal. - extract_refined_covspans(mir_body, hir_info, basic_coverage_blocks, &mut code_mappings); + extract_refined_covspans(mir_body, hir_info, graph, &mut code_mappings); } - branch_pairs.extend(extract_branch_pairs(mir_body, hir_info, basic_coverage_blocks)); + branch_pairs.extend(extract_branch_pairs(mir_body, hir_info, graph)); extract_mcdc_mappings( mir_body, tcx, hir_info.body_span, - basic_coverage_blocks, + graph, &mut mcdc_bitmap_bits, &mut mcdc_degraded_branches, &mut mcdc_mappings, ); ExtractedMappings { - num_bcbs: basic_coverage_blocks.num_nodes(), + num_bcbs: graph.num_nodes(), code_mappings, branch_pairs, mcdc_bitmap_bits, @@ -211,7 +211,7 @@ fn resolve_block_markers( pub(super) fn extract_branch_pairs( mir_body: &mir::Body<'_>, hir_info: &ExtractedHirInfo, - basic_coverage_blocks: &CoverageGraph, + graph: &CoverageGraph, ) -> Vec<BranchPair> { let Some(coverage_info_hi) = mir_body.coverage_info_hi.as_deref() else { return vec![] }; @@ -228,8 +228,7 @@ pub(super) fn extract_branch_pairs( } let span = unexpand_into_body_span(raw_span, hir_info.body_span)?; - let bcb_from_marker = - |marker: BlockMarkerId| basic_coverage_blocks.bcb_from_bb(block_markers[marker]?); + let bcb_from_marker = |marker: BlockMarkerId| graph.bcb_from_bb(block_markers[marker]?); let true_bcb = bcb_from_marker(true_marker)?; let false_bcb = bcb_from_marker(false_marker)?; @@ -243,7 +242,7 @@ pub(super) fn extract_mcdc_mappings( mir_body: &mir::Body<'_>, tcx: TyCtxt<'_>, body_span: Span, - basic_coverage_blocks: &CoverageGraph, + graph: &CoverageGraph, mcdc_bitmap_bits: &mut usize, mcdc_degraded_branches: &mut impl Extend<MCDCBranch>, mcdc_mappings: &mut impl Extend<(MCDCDecision, Vec<MCDCBranch>)>, @@ -252,8 +251,7 @@ pub(super) fn extract_mcdc_mappings( let block_markers = resolve_block_markers(coverage_info_hi, mir_body); - let bcb_from_marker = - |marker: BlockMarkerId| basic_coverage_blocks.bcb_from_bb(block_markers[marker]?); + let bcb_from_marker = |marker: BlockMarkerId| graph.bcb_from_bb(block_markers[marker]?); let check_branch_bcb = |raw_span: Span, true_marker: BlockMarkerId, false_marker: BlockMarkerId| { diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs index 2e4c503f3ce..57956448414 100644 --- a/compiler/rustc_mir_transform/src/coverage/mod.rs +++ b/compiler/rustc_mir_transform/src/coverage/mod.rs @@ -13,19 +13,18 @@ use rustc_hir::intravisit::{Visitor, walk_expr}; use rustc_middle::hir::map::Map; use rustc_middle::hir::nested_filter; use rustc_middle::mir::coverage::{ - CoverageKind, DecisionInfo, FunctionCoverageInfo, Mapping, MappingKind, SourceRegion, + CoverageKind, DecisionInfo, FunctionCoverageInfo, Mapping, MappingKind, }; use rustc_middle::mir::{ self, BasicBlock, BasicBlockData, SourceInfo, Statement, StatementKind, Terminator, TerminatorKind, }; use rustc_middle::ty::TyCtxt; +use rustc_span::Span; use rustc_span::def_id::LocalDefId; -use rustc_span::source_map::SourceMap; -use rustc_span::{BytePos, Pos, RelativeBytePos, Span, Symbol}; -use tracing::{debug, debug_span, instrument, trace}; +use tracing::{debug, debug_span, trace}; -use crate::coverage::counters::{CounterIncrementSite, CoverageCounters}; +use crate::coverage::counters::{CoverageCounters, Site}; use crate::coverage::graph::CoverageGraph; use crate::coverage::mappings::ExtractedMappings; @@ -72,16 +71,15 @@ fn instrument_function_for_coverage<'tcx>(tcx: TyCtxt<'tcx>, mir_body: &mut mir: let _span = debug_span!("instrument_function_for_coverage", ?def_id).entered(); let hir_info = extract_hir_info(tcx, def_id.expect_local()); - let basic_coverage_blocks = CoverageGraph::from_mir(mir_body); + + // Build the coverage graph, which is a simplified view of the MIR control-flow + // graph that ignores some details not relevant to coverage instrumentation. + let graph = CoverageGraph::from_mir(mir_body); //////////////////////////////////////////////////// // Extract coverage spans and other mapping info from MIR. - let extracted_mappings = mappings::extract_all_mapping_info_from_mir( - tcx, - mir_body, - &hir_info, - &basic_coverage_blocks, - ); + let extracted_mappings = + mappings::extract_all_mapping_info_from_mir(tcx, mir_body, &hir_info, &graph); //////////////////////////////////////////////////// // Create an optimized mix of `Counter`s and `Expression`s for the `CoverageGraph`. Ensure @@ -95,23 +93,18 @@ fn instrument_function_for_coverage<'tcx>(tcx: TyCtxt<'tcx>, mir_body: &mut mir: } let coverage_counters = - CoverageCounters::make_bcb_counters(&basic_coverage_blocks, &bcbs_with_counter_mappings); + CoverageCounters::make_bcb_counters(&graph, &bcbs_with_counter_mappings); - let mappings = create_mappings(tcx, &hir_info, &extracted_mappings, &coverage_counters); + let mappings = create_mappings(&extracted_mappings, &coverage_counters); if mappings.is_empty() { // No spans could be converted into valid mappings, so skip this function. debug!("no spans could be converted into valid mappings; skipping"); return; } - inject_coverage_statements( - mir_body, - &basic_coverage_blocks, - &extracted_mappings, - &coverage_counters, - ); + inject_coverage_statements(mir_body, &graph, &extracted_mappings, &coverage_counters); - inject_mcdc_statements(mir_body, &basic_coverage_blocks, &extracted_mappings); + inject_mcdc_statements(mir_body, &graph, &extracted_mappings); let mcdc_num_condition_bitmaps = extracted_mappings .mcdc_mappings @@ -122,6 +115,7 @@ fn instrument_function_for_coverage<'tcx>(tcx: TyCtxt<'tcx>, mir_body: &mut mir: mir_body.function_coverage_info = Some(Box::new(FunctionCoverageInfo { function_source_hash: hir_info.function_source_hash, + body_span: hir_info.body_span, num_counters: coverage_counters.num_counters(), mcdc_bitmap_bits: extracted_mappings.mcdc_bitmap_bits, expressions: coverage_counters.into_expressions(), @@ -135,26 +129,12 @@ fn instrument_function_for_coverage<'tcx>(tcx: TyCtxt<'tcx>, mir_body: &mut mir: /// /// Precondition: All BCBs corresponding to those spans have been given /// coverage counters. -fn create_mappings<'tcx>( - tcx: TyCtxt<'tcx>, - hir_info: &ExtractedHirInfo, +fn create_mappings( extracted_mappings: &ExtractedMappings, coverage_counters: &CoverageCounters, ) -> Vec<Mapping> { - let source_map = tcx.sess.source_map(); - let body_span = hir_info.body_span; - - let source_file = source_map.lookup_source_file(body_span.lo()); - - use rustc_session::RemapFileNameExt; - use rustc_session::config::RemapPathScopeComponents; - let file_name = Symbol::intern( - &source_file.name.for_scope(tcx.sess, RemapPathScopeComponents::MACRO).to_string_lossy(), - ); - let term_for_bcb = |bcb| coverage_counters.term_for_bcb(bcb).expect("all BCBs with spans were given counters"); - let region_for_span = |span: Span| make_source_region(source_map, file_name, span, body_span); // Fully destructure the mappings struct to make sure we don't miss any kinds. let ExtractedMappings { @@ -167,22 +147,20 @@ fn create_mappings<'tcx>( } = extracted_mappings; let mut mappings = Vec::new(); - mappings.extend(code_mappings.iter().filter_map( + mappings.extend(code_mappings.iter().map( // Ordinary code mappings are the simplest kind. |&mappings::CodeMapping { span, bcb }| { - let source_region = region_for_span(span)?; let kind = MappingKind::Code(term_for_bcb(bcb)); - Some(Mapping { kind, source_region }) + Mapping { kind, span } }, )); - mappings.extend(branch_pairs.iter().filter_map( + mappings.extend(branch_pairs.iter().map( |&mappings::BranchPair { span, true_bcb, false_bcb }| { let true_term = term_for_bcb(true_bcb); let false_term = term_for_bcb(false_bcb); let kind = MappingKind::Branch { true_term, false_term }; - let source_region = region_for_span(span)?; - Some(Mapping { kind, source_region }) + Mapping { kind, span } }, )); @@ -190,7 +168,7 @@ fn create_mappings<'tcx>( |bcb| coverage_counters.term_for_bcb(bcb).expect("all BCBs with spans were given counters"); // MCDC branch mappings are appended with their decisions in case decisions were ignored. - mappings.extend(mcdc_degraded_branches.iter().filter_map( + mappings.extend(mcdc_degraded_branches.iter().map( |&mappings::MCDCBranch { span, true_bcb, @@ -199,10 +177,9 @@ fn create_mappings<'tcx>( true_index: _, false_index: _, }| { - let source_region = region_for_span(span)?; let true_term = term_for_bcb(true_bcb); let false_term = term_for_bcb(false_bcb); - Some(Mapping { kind: MappingKind::Branch { true_term, false_term }, source_region }) + Mapping { kind: MappingKind::Branch { true_term, false_term }, span } }, )); @@ -210,7 +187,7 @@ fn create_mappings<'tcx>( let num_conditions = branches.len() as u16; let conditions = branches .into_iter() - .filter_map( + .map( |&mappings::MCDCBranch { span, true_bcb, @@ -219,32 +196,28 @@ fn create_mappings<'tcx>( true_index: _, false_index: _, }| { - let source_region = region_for_span(span)?; let true_term = term_for_bcb(true_bcb); let false_term = term_for_bcb(false_bcb); - Some(Mapping { + Mapping { kind: MappingKind::MCDCBranch { true_term, false_term, mcdc_params: condition_info, }, - source_region, - }) + span, + } }, ) .collect::<Vec<_>>(); - if conditions.len() == num_conditions as usize - && let Some(source_region) = region_for_span(decision.span) - { + if conditions.len() == num_conditions as usize { // LLVM requires end index for counter mapping regions. let kind = MappingKind::MCDCDecision(DecisionInfo { bitmap_idx: (decision.bitmap_idx + decision.num_test_vectors) as u32, num_conditions, }); - mappings.extend( - std::iter::once(Mapping { kind, source_region }).chain(conditions.into_iter()), - ); + let span = decision.span; + mappings.extend(std::iter::once(Mapping { kind, span }).chain(conditions.into_iter())); } else { mappings.extend(conditions.into_iter().map(|mapping| { let MappingKind::MCDCBranch { true_term, false_term, mcdc_params: _ } = @@ -252,10 +225,7 @@ fn create_mappings<'tcx>( else { unreachable!("all mappings here are MCDCBranch as shown above"); }; - Mapping { - kind: MappingKind::Branch { true_term, false_term }, - source_region: mapping.source_region, - } + Mapping { kind: MappingKind::Branch { true_term, false_term }, span: mapping.span } })) } } @@ -267,22 +237,22 @@ fn create_mappings<'tcx>( /// inject any necessary coverage statements into MIR. fn inject_coverage_statements<'tcx>( mir_body: &mut mir::Body<'tcx>, - basic_coverage_blocks: &CoverageGraph, + graph: &CoverageGraph, extracted_mappings: &ExtractedMappings, coverage_counters: &CoverageCounters, ) { // Inject counter-increment statements into MIR. - for (id, counter_increment_site) in coverage_counters.counter_increment_sites() { + for (id, site) in coverage_counters.counter_increment_sites() { // Determine the block to inject a counter-increment statement into. // For BCB nodes this is just their first block, but for edges we need // to create a new block between the two BCBs, and inject into that. - let target_bb = match *counter_increment_site { - CounterIncrementSite::Node { bcb } => basic_coverage_blocks[bcb].leader_bb(), - CounterIncrementSite::Edge { from_bcb, to_bcb } => { + let target_bb = match site { + Site::Node { bcb } => graph[bcb].leader_bb(), + Site::Edge { from_bcb, to_bcb } => { // Create a new block between the last block of `from_bcb` and // the first block of `to_bcb`. - let from_bb = basic_coverage_blocks[from_bcb].last_bb(); - let to_bb = basic_coverage_blocks[to_bcb].leader_bb(); + let from_bb = graph[from_bcb].last_bb(); + let to_bb = graph[to_bcb].leader_bb(); let new_bb = inject_edge_counter_basic_block(mir_body, from_bb, to_bb); debug!( @@ -315,7 +285,7 @@ fn inject_coverage_statements<'tcx>( inject_statement( mir_body, CoverageKind::ExpressionUsed { id: expression_id }, - basic_coverage_blocks[bcb].leader_bb(), + graph[bcb].leader_bb(), ); } } @@ -324,13 +294,13 @@ fn inject_coverage_statements<'tcx>( /// For each decision inject statements to update test vector bitmap after it has been evaluated. fn inject_mcdc_statements<'tcx>( mir_body: &mut mir::Body<'tcx>, - basic_coverage_blocks: &CoverageGraph, + graph: &CoverageGraph, extracted_mappings: &ExtractedMappings, ) { for (decision, conditions) in &extracted_mappings.mcdc_mappings { // Inject test vector update first because `inject_statement` always insert new statement at head. for &end in &decision.end_bcbs { - let end_bb = basic_coverage_blocks[end].leader_bb(); + let end_bb = graph[end].leader_bb(); inject_statement( mir_body, CoverageKind::TestVectorBitmapUpdate { @@ -351,7 +321,7 @@ fn inject_mcdc_statements<'tcx>( } in conditions { for (index, bcb) in [(false_index, false_bcb), (true_index, true_bcb)] { - let bb = basic_coverage_blocks[bcb].leader_bb(); + let bb = graph[bcb].leader_bb(); inject_statement( mir_body, CoverageKind::CondBitmapUpdate { @@ -398,114 +368,6 @@ fn inject_statement(mir_body: &mut mir::Body<'_>, counter_kind: CoverageKind, bb data.statements.insert(0, statement); } -/// Convert the Span into its file name, start line and column, and end line and column. -/// -/// Line numbers and column numbers are 1-based. Unlike most column numbers emitted by -/// the compiler, these column numbers are denoted in **bytes**, because that's what -/// LLVM's `llvm-cov` tool expects to see in coverage maps. -/// -/// Returns `None` if the conversion failed for some reason. This shouldn't happen, -/// but it's hard to rule out entirely (especially in the presence of complex macros -/// or other expansions), and if it does happen then skipping a span or function is -/// better than an ICE or `llvm-cov` failure that the user might have no way to avoid. -#[instrument(level = "debug", skip(source_map))] -fn make_source_region( - source_map: &SourceMap, - file_name: Symbol, - span: Span, - body_span: Span, -) -> Option<SourceRegion> { - let lo = span.lo(); - let hi = span.hi(); - - let file = source_map.lookup_source_file(lo); - if !file.contains(hi) { - debug!(?span, ?file, ?lo, ?hi, "span crosses multiple files; skipping"); - return None; - } - - // Column numbers need to be in bytes, so we can't use the more convenient - // `SourceMap` methods for looking up file coordinates. - let rpos_and_line_and_byte_column = |pos: BytePos| -> Option<(RelativeBytePos, usize, usize)> { - let rpos = file.relative_position(pos); - let line_index = file.lookup_line(rpos)?; - let line_start = file.lines()[line_index]; - // Line numbers and column numbers are 1-based, so add 1 to each. - Some((rpos, line_index + 1, (rpos - line_start).to_usize() + 1)) - }; - - let (lo_rpos, mut start_line, mut start_col) = rpos_and_line_and_byte_column(lo)?; - let (hi_rpos, mut end_line, mut end_col) = rpos_and_line_and_byte_column(hi)?; - - // If the span is empty, try to expand it horizontally by one character's - // worth of bytes, so that it is more visible in `llvm-cov` reports. - // We do this after resolving line/column numbers, so that empty spans at the - // end of a line get an extra column instead of wrapping to the next line. - if span.is_empty() - && body_span.contains(span) - && let Some(src) = &file.src - { - // Prefer to expand the end position, if it won't go outside the body span. - if hi < body_span.hi() { - let hi_rpos = hi_rpos.to_usize(); - let nudge_bytes = src.ceil_char_boundary(hi_rpos + 1) - hi_rpos; - end_col += nudge_bytes; - } else if lo > body_span.lo() { - let lo_rpos = lo_rpos.to_usize(); - let nudge_bytes = lo_rpos - src.floor_char_boundary(lo_rpos - 1); - // Subtract the nudge, but don't go below column 1. - start_col = start_col.saturating_sub(nudge_bytes).max(1); - } - // If neither nudge could be applied, stick with the empty span coordinates. - } - - // Apply an offset so that code in doctests has correct line numbers. - // FIXME(#79417): Currently we have no way to offset doctest _columns_. - start_line = source_map.doctest_offset_line(&file.name, start_line); - end_line = source_map.doctest_offset_line(&file.name, end_line); - - check_source_region(SourceRegion { - file_name, - start_line: start_line as u32, - start_col: start_col as u32, - end_line: end_line as u32, - end_col: end_col as u32, - }) -} - -/// If `llvm-cov` sees a source region that is improperly ordered (end < start), -/// it will immediately exit with a fatal error. To prevent that from happening, -/// discard regions that are improperly ordered, or might be interpreted in a -/// way that makes them improperly ordered. -fn check_source_region(source_region: SourceRegion) -> Option<SourceRegion> { - let SourceRegion { file_name: _, start_line, start_col, end_line, end_col } = source_region; - - // Line/column coordinates are supposed to be 1-based. If we ever emit - // coordinates of 0, `llvm-cov` might misinterpret them. - let all_nonzero = [start_line, start_col, end_line, end_col].into_iter().all(|x| x != 0); - // Coverage mappings use the high bit of `end_col` to indicate that a - // region is actually a "gap" region, so make sure it's unset. - let end_col_has_high_bit_unset = (end_col & (1 << 31)) == 0; - // If a region is improperly ordered (end < start), `llvm-cov` will exit - // with a fatal error, which is inconvenient for users and hard to debug. - let is_ordered = (start_line, start_col) <= (end_line, end_col); - - if all_nonzero && end_col_has_high_bit_unset && is_ordered { - Some(source_region) - } else { - debug!( - ?source_region, - ?all_nonzero, - ?end_col_has_high_bit_unset, - ?is_ordered, - "Skipping source region that would be misinterpreted or rejected by LLVM" - ); - // If this happens in a debug build, ICE to make it easier to notice. - debug_assert!(false, "Improper source region: {source_region:?}"); - None - } -} - /// Function information extracted from HIR by the coverage instrumentor. #[derive(Debug)] struct ExtractedHirInfo { diff --git a/compiler/rustc_mir_transform/src/coverage/query.rs b/compiler/rustc_mir_transform/src/coverage/query.rs index df151f8cca3..edaec3c7965 100644 --- a/compiler/rustc_mir_transform/src/coverage/query.rs +++ b/compiler/rustc_mir_transform/src/coverage/query.rs @@ -1,7 +1,11 @@ use rustc_data_structures::captures::Captures; +use rustc_index::bit_set::BitSet; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::mir::coverage::{CounterId, CoverageKind}; -use rustc_middle::mir::{Body, CoverageIdsInfo, Statement, StatementKind}; +use rustc_middle::mir::coverage::{ + CounterId, CovTerm, CoverageIdsInfo, CoverageKind, Expression, ExpressionId, + FunctionCoverageInfo, MappingKind, Op, +}; +use rustc_middle::mir::{Body, Statement, StatementKind}; use rustc_middle::query::TyCtxtAt; use rustc_middle::ty::{self, TyCtxt}; use rustc_middle::util::Providers; @@ -86,15 +90,46 @@ fn coverage_ids_info<'tcx>( ) -> CoverageIdsInfo { let mir_body = tcx.instance_mir(instance_def); - let max_counter_id = all_coverage_in_mir_body(mir_body) - .filter_map(|kind| match *kind { - CoverageKind::CounterIncrement { id } => Some(id), - _ => None, - }) - .max() - .unwrap_or(CounterId::ZERO); + let Some(fn_cov_info) = mir_body.function_coverage_info.as_deref() else { + return CoverageIdsInfo { + counters_seen: BitSet::new_empty(0), + zero_expressions: BitSet::new_empty(0), + }; + }; + + let mut counters_seen = BitSet::new_empty(fn_cov_info.num_counters); + let mut expressions_seen = BitSet::new_filled(fn_cov_info.expressions.len()); + + // For each expression ID that is directly used by one or more mappings, + // mark it as not-yet-seen. This indicates that we expect to see a + // corresponding `ExpressionUsed` statement during MIR traversal. + for mapping in fn_cov_info.mappings.iter() { + // Currently we only worry about ordinary code mappings. + // For branch and MC/DC mappings, expressions might not correspond + // to any particular point in the control-flow graph. + // (Keep this in sync with the injection of `ExpressionUsed` + // statements in the `InstrumentCoverage` MIR pass.) + if let MappingKind::Code(CovTerm::Expression(id)) = mapping.kind { + expressions_seen.remove(id); + } + } - CoverageIdsInfo { max_counter_id } + for kind in all_coverage_in_mir_body(mir_body) { + match *kind { + CoverageKind::CounterIncrement { id } => { + counters_seen.insert(id); + } + CoverageKind::ExpressionUsed { id } => { + expressions_seen.insert(id); + } + _ => {} + } + } + + let zero_expressions = + identify_zero_expressions(fn_cov_info, &counters_seen, &expressions_seen); + + CoverageIdsInfo { counters_seen, zero_expressions } } fn all_coverage_in_mir_body<'a, 'tcx>( @@ -112,3 +147,94 @@ fn is_inlined(body: &Body<'_>, statement: &Statement<'_>) -> bool { let scope_data = &body.source_scopes[statement.source_info.scope]; scope_data.inlined.is_some() || scope_data.inlined_parent_scope.is_some() } + +/// Identify expressions that will always have a value of zero, and note +/// their IDs in a `BitSet`. Mappings that refer to a zero expression +/// can instead become mappings to a constant zero value. +/// +/// This function mainly exists to preserve the simplifications that were +/// already being performed by the Rust-side expression renumbering, so that +/// the resulting coverage mappings don't get worse. +fn identify_zero_expressions( + fn_cov_info: &FunctionCoverageInfo, + counters_seen: &BitSet<CounterId>, + expressions_seen: &BitSet<ExpressionId>, +) -> BitSet<ExpressionId> { + // The set of expressions that either were optimized out entirely, or + // have zero as both of their operands, and will therefore always have + // a value of zero. Other expressions that refer to these as operands + // can have those operands replaced with `CovTerm::Zero`. + let mut zero_expressions = BitSet::new_empty(fn_cov_info.expressions.len()); + + // Simplify a copy of each expression based on lower-numbered expressions, + // and then update the set of always-zero expressions if necessary. + // (By construction, expressions can only refer to other expressions + // that have lower IDs, so one pass is sufficient.) + for (id, expression) in fn_cov_info.expressions.iter_enumerated() { + if !expressions_seen.contains(id) { + // If an expression was not seen, it must have been optimized away, + // so any operand that refers to it can be replaced with zero. + zero_expressions.insert(id); + continue; + } + + // We don't need to simplify the actual expression data in the + // expressions list; we can just simplify a temporary copy and then + // use that to update the set of always-zero expressions. + let Expression { mut lhs, op, mut rhs } = *expression; + + // If an expression has an operand that is also an expression, the + // operand's ID must be strictly lower. This is what lets us find + // all zero expressions in one pass. + let assert_operand_expression_is_lower = |operand_id: ExpressionId| { + assert!( + operand_id < id, + "Operand {operand_id:?} should be less than {id:?} in {expression:?}", + ) + }; + + // If an operand refers to a counter or expression that is always + // zero, then that operand can be replaced with `CovTerm::Zero`. + let maybe_set_operand_to_zero = |operand: &mut CovTerm| { + if let CovTerm::Expression(id) = *operand { + assert_operand_expression_is_lower(id); + } + + if is_zero_term(&counters_seen, &zero_expressions, *operand) { + *operand = CovTerm::Zero; + } + }; + maybe_set_operand_to_zero(&mut lhs); + maybe_set_operand_to_zero(&mut rhs); + + // Coverage counter values cannot be negative, so if an expression + // involves subtraction from zero, assume that its RHS must also be zero. + // (Do this after simplifications that could set the LHS to zero.) + if lhs == CovTerm::Zero && op == Op::Subtract { + rhs = CovTerm::Zero; + } + + // After the above simplifications, if both operands are zero, then + // we know that this expression is always zero too. + if lhs == CovTerm::Zero && rhs == CovTerm::Zero { + zero_expressions.insert(id); + } + } + + zero_expressions +} + +/// Returns `true` if the given term is known to have a value of zero, taking +/// into account knowledge of which counters are unused and which expressions +/// are always zero. +fn is_zero_term( + counters_seen: &BitSet<CounterId>, + zero_expressions: &BitSet<ExpressionId>, + term: CovTerm, +) -> bool { + match term { + CovTerm::Zero => true, + CovTerm::Counter(id) => !counters_seen.contains(id), + CovTerm::Expression(id) => zero_expressions.contains(id), + } +} diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs index 085c738f1f9..314a86ea52f 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans.rs @@ -17,14 +17,13 @@ mod from_mir; pub(super) fn extract_refined_covspans( mir_body: &mir::Body<'_>, hir_info: &ExtractedHirInfo, - basic_coverage_blocks: &CoverageGraph, + graph: &CoverageGraph, code_mappings: &mut impl Extend<mappings::CodeMapping>, ) { - let ExtractedCovspans { mut covspans } = - extract_covspans_from_mir(mir_body, hir_info, basic_coverage_blocks); + let ExtractedCovspans { mut covspans } = extract_covspans_from_mir(mir_body, hir_info, graph); // First, perform the passes that need macro information. - covspans.sort_by(|a, b| basic_coverage_blocks.cmp_in_dominator_order(a.bcb, b.bcb)); + covspans.sort_by(|a, b| graph.cmp_in_dominator_order(a.bcb, b.bcb)); remove_unwanted_expansion_spans(&mut covspans); split_visible_macro_spans(&mut covspans); @@ -34,7 +33,7 @@ pub(super) fn extract_refined_covspans( let compare_covspans = |a: &Covspan, b: &Covspan| { compare_spans(a.span, b.span) // After deduplication, we want to keep only the most-dominated BCB. - .then_with(|| basic_coverage_blocks.cmp_in_dominator_order(a.bcb, b.bcb).reverse()) + .then_with(|| graph.cmp_in_dominator_order(a.bcb, b.bcb).reverse()) }; covspans.sort_by(compare_covspans); diff --git a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs index 875db23ce09..26ce743be36 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs @@ -22,13 +22,13 @@ pub(crate) struct ExtractedCovspans { pub(crate) fn extract_covspans_from_mir( mir_body: &mir::Body<'_>, hir_info: &ExtractedHirInfo, - basic_coverage_blocks: &CoverageGraph, + graph: &CoverageGraph, ) -> ExtractedCovspans { let &ExtractedHirInfo { body_span, .. } = hir_info; let mut covspans = vec![]; - for (bcb, bcb_data) in basic_coverage_blocks.iter_enumerated() { + for (bcb, bcb_data) in graph.iter_enumerated() { bcb_to_initial_coverage_spans(mir_body, body_span, bcb, bcb_data, &mut covspans); } @@ -97,6 +97,7 @@ fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span> { StatementKind::StorageLive(_) | StatementKind::StorageDead(_) | StatementKind::ConstEvalCounter + | StatementKind::BackwardIncompatibleDropHint { .. } | StatementKind::Nop => None, // FIXME(#78546): MIR InstrumentCoverage - Can the source_info.span for `FakeRead` diff --git a/compiler/rustc_mir_transform/src/coverage/tests.rs b/compiler/rustc_mir_transform/src/coverage/tests.rs index 233ca9981c5..b2ee50de50a 100644 --- a/compiler/rustc_mir_transform/src/coverage/tests.rs +++ b/compiler/rustc_mir_transform/src/coverage/tests.rs @@ -223,16 +223,12 @@ fn print_mir_graphviz(name: &str, mir_body: &Body<'_>) { } } -fn print_coverage_graphviz( - name: &str, - mir_body: &Body<'_>, - basic_coverage_blocks: &graph::CoverageGraph, -) { +fn print_coverage_graphviz(name: &str, mir_body: &Body<'_>, graph: &graph::CoverageGraph) { if PRINT_GRAPHS { println!( "digraph {} {{\n{}\n}}", name, - basic_coverage_blocks + graph .iter_enumerated() .map(|(bcb, bcb_data)| { format!( @@ -240,7 +236,7 @@ fn print_coverage_graphviz( bcb, bcb, mir_body[bcb_data.last_bb()].terminator().kind.name(), - basic_coverage_blocks + graph .successors(bcb) .map(|successor| { format!(" {:?} -> {:?};", bcb, successor) }) .join("\n") @@ -300,11 +296,11 @@ fn goto_switchint<'a>() -> Body<'a> { #[track_caller] fn assert_successors( - basic_coverage_blocks: &graph::CoverageGraph, + graph: &graph::CoverageGraph, bcb: BasicCoverageBlock, expected_successors: &[BasicCoverageBlock], ) { - let mut successors = basic_coverage_blocks.successors[bcb].clone(); + let mut successors = graph.successors[bcb].clone(); successors.sort_unstable(); assert_eq!(successors, expected_successors); } @@ -315,8 +311,8 @@ fn test_covgraph_goto_switchint() { if false { eprintln!("basic_blocks = {}", debug_basic_blocks(&mir_body)); } - let basic_coverage_blocks = graph::CoverageGraph::from_mir(&mir_body); - print_coverage_graphviz("covgraph_goto_switchint ", &mir_body, &basic_coverage_blocks); + let graph = graph::CoverageGraph::from_mir(&mir_body); + print_coverage_graphviz("covgraph_goto_switchint ", &mir_body, &graph); /* ┌──────────────┐ ┌─────────────────┐ │ bcb2: Return │ ◀── │ bcb0: SwitchInt │ @@ -328,16 +324,11 @@ fn test_covgraph_goto_switchint() { │ bcb1: Return │ └─────────────────┘ */ - assert_eq!( - basic_coverage_blocks.num_nodes(), - 3, - "basic_coverage_blocks: {:?}", - basic_coverage_blocks.iter_enumerated().collect::<Vec<_>>() - ); + assert_eq!(graph.num_nodes(), 3, "graph: {:?}", graph.iter_enumerated().collect::<Vec<_>>()); - assert_successors(&basic_coverage_blocks, bcb(0), &[bcb(1), bcb(2)]); - assert_successors(&basic_coverage_blocks, bcb(1), &[]); - assert_successors(&basic_coverage_blocks, bcb(2), &[]); + assert_successors(&graph, bcb(0), &[bcb(1), bcb(2)]); + assert_successors(&graph, bcb(1), &[]); + assert_successors(&graph, bcb(2), &[]); } /// Create a mock `Body` with a loop. @@ -383,12 +374,8 @@ fn switchint_then_loop_else_return<'a>() -> Body<'a> { #[test] fn test_covgraph_switchint_then_loop_else_return() { let mir_body = switchint_then_loop_else_return(); - let basic_coverage_blocks = graph::CoverageGraph::from_mir(&mir_body); - print_coverage_graphviz( - "covgraph_switchint_then_loop_else_return", - &mir_body, - &basic_coverage_blocks, - ); + let graph = graph::CoverageGraph::from_mir(&mir_body); + print_coverage_graphviz("covgraph_switchint_then_loop_else_return", &mir_body, &graph); /* ┌─────────────────┐ │ bcb0: Call │ @@ -408,17 +395,12 @@ fn test_covgraph_switchint_then_loop_else_return() { │ │ └─────────────────────────────────────┘ */ - assert_eq!( - basic_coverage_blocks.num_nodes(), - 4, - "basic_coverage_blocks: {:?}", - basic_coverage_blocks.iter_enumerated().collect::<Vec<_>>() - ); + assert_eq!(graph.num_nodes(), 4, "graph: {:?}", graph.iter_enumerated().collect::<Vec<_>>()); - assert_successors(&basic_coverage_blocks, bcb(0), &[bcb(1)]); - assert_successors(&basic_coverage_blocks, bcb(1), &[bcb(2), bcb(3)]); - assert_successors(&basic_coverage_blocks, bcb(2), &[]); - assert_successors(&basic_coverage_blocks, bcb(3), &[bcb(1)]); + assert_successors(&graph, bcb(0), &[bcb(1)]); + assert_successors(&graph, bcb(1), &[bcb(2), bcb(3)]); + assert_successors(&graph, bcb(2), &[]); + assert_successors(&graph, bcb(3), &[bcb(1)]); } /// Create a mock `Body` with nested loops. @@ -494,11 +476,11 @@ fn switchint_loop_then_inner_loop_else_break<'a>() -> Body<'a> { #[test] fn test_covgraph_switchint_loop_then_inner_loop_else_break() { let mir_body = switchint_loop_then_inner_loop_else_break(); - let basic_coverage_blocks = graph::CoverageGraph::from_mir(&mir_body); + let graph = graph::CoverageGraph::from_mir(&mir_body); print_coverage_graphviz( "covgraph_switchint_loop_then_inner_loop_else_break", &mir_body, - &basic_coverage_blocks, + &graph, ); /* ┌─────────────────┐ @@ -531,18 +513,13 @@ fn test_covgraph_switchint_loop_then_inner_loop_else_break() { │ │ └────────────────────────────────────────────┘ */ - assert_eq!( - basic_coverage_blocks.num_nodes(), - 7, - "basic_coverage_blocks: {:?}", - basic_coverage_blocks.iter_enumerated().collect::<Vec<_>>() - ); - - assert_successors(&basic_coverage_blocks, bcb(0), &[bcb(1)]); - assert_successors(&basic_coverage_blocks, bcb(1), &[bcb(2), bcb(3)]); - assert_successors(&basic_coverage_blocks, bcb(2), &[]); - assert_successors(&basic_coverage_blocks, bcb(3), &[bcb(4)]); - assert_successors(&basic_coverage_blocks, bcb(4), &[bcb(5), bcb(6)]); - assert_successors(&basic_coverage_blocks, bcb(5), &[bcb(1)]); - assert_successors(&basic_coverage_blocks, bcb(6), &[bcb(4)]); + assert_eq!(graph.num_nodes(), 7, "graph: {:?}", graph.iter_enumerated().collect::<Vec<_>>()); + + assert_successors(&graph, bcb(0), &[bcb(1)]); + assert_successors(&graph, bcb(1), &[bcb(2), bcb(3)]); + assert_successors(&graph, bcb(2), &[]); + assert_successors(&graph, bcb(3), &[bcb(4)]); + assert_successors(&graph, bcb(4), &[bcb(5), bcb(6)]); + assert_successors(&graph, bcb(5), &[bcb(1)]); + assert_successors(&graph, bcb(6), &[bcb(4)]); } diff --git a/compiler/rustc_mir_transform/src/cross_crate_inline.rs b/compiler/rustc_mir_transform/src/cross_crate_inline.rs index 42cbece32d8..e1f1dd83f0d 100644 --- a/compiler/rustc_mir_transform/src/cross_crate_inline.rs +++ b/compiler/rustc_mir_transform/src/cross_crate_inline.rs @@ -1,4 +1,4 @@ -use rustc_attr::InlineAttr; +use rustc_attr_parsing::InlineAttr; use rustc_hir::def::DefKind; use rustc_hir::def_id::LocalDefId; use rustc_middle::mir::visit::Visitor; @@ -50,6 +50,16 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { _ => {} } + let sig = tcx.fn_sig(def_id).instantiate_identity(); + for ty in sig.inputs().skip_binder().iter().chain(std::iter::once(&sig.output().skip_binder())) + { + // FIXME(f16_f128): in order to avoid crashes building `core`, always inline to skip + // codegen if the function is not used. + if ty == &tcx.types.f16 || ty == &tcx.types.f128 { + return true; + } + } + // Don't do any inference when incremental compilation is enabled; the additional inlining that // inference permits also creates more work for small edits. if tcx.sess.opts.incremental.is_some() { diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs index 7d073f1fa57..711cf2edc46 100644 --- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs +++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs @@ -16,7 +16,7 @@ use rustc_middle::bug; use rustc_middle::mir::interpret::{InterpResult, Scalar}; use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor}; use rustc_middle::mir::*; -use rustc_middle::ty::layout::{HasParamEnv, LayoutOf}; +use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_mir_dataflow::fmt::DebugWithContext; use rustc_mir_dataflow::lattice::{FlatSet, HasBottom}; @@ -82,7 +82,7 @@ struct ConstAnalysis<'a, 'tcx> { tcx: TyCtxt<'tcx>, local_decls: &'a LocalDecls<'tcx>, ecx: InterpCx<'tcx, DummyMachine>, - param_env: ty::ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, } impl<'tcx> Analysis<'tcx> for ConstAnalysis<'_, 'tcx> { @@ -106,7 +106,7 @@ impl<'tcx> Analysis<'tcx> for ConstAnalysis<'_, 'tcx> { } } - fn apply_statement_effect( + fn apply_primary_statement_effect( &mut self, state: &mut Self::Domain, statement: &Statement<'tcx>, @@ -117,7 +117,7 @@ impl<'tcx> Analysis<'tcx> for ConstAnalysis<'_, 'tcx> { } } - fn apply_terminator_effect<'mir>( + fn apply_primary_terminator_effect<'mir>( &mut self, state: &mut Self::Domain, terminator: &'mir Terminator<'tcx>, @@ -144,13 +144,13 @@ impl<'tcx> Analysis<'tcx> for ConstAnalysis<'_, 'tcx> { impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { fn new(tcx: TyCtxt<'tcx>, body: &'a Body<'tcx>, map: Map<'tcx>) -> Self { - let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); + let typing_env = body.typing_env(tcx); Self { map, tcx, local_decls: &body.local_decls, - ecx: InterpCx::new(tcx, DUMMY_SP, param_env, DummyMachine), - param_env, + ecx: InterpCx::new(tcx, DUMMY_SP, typing_env, DummyMachine), + typing_env, } } @@ -186,7 +186,8 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { | StatementKind::FakeRead(..) | StatementKind::PlaceMention(..) | StatementKind::Coverage(..) - | StatementKind::AscribeUserType(..) => (), + | StatementKind::BackwardIncompatibleDropHint { .. } + | StatementKind::AscribeUserType(..) => {} } } @@ -223,7 +224,7 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { } /// The effect of a successful function call return should not be - /// applied here, see [`Analysis::apply_terminator_effect`]. + /// applied here, see [`Analysis::apply_primary_terminator_effect`]. fn handle_terminator<'mir>( &self, terminator: &'mir Terminator<'tcx>, @@ -389,7 +390,7 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { && let Some(operand_ty) = operand_ty.builtin_deref(true) && let ty::Array(_, len) = operand_ty.kind() && let Some(len) = Const::Ty(self.tcx.types.usize, *len) - .try_eval_scalar_int(self.tcx, self.param_env) + .try_eval_scalar_int(self.tcx, self.typing_env) { state.insert_value_idx(target_len, FlatSet::Elem(len.into()), &self.map); } @@ -411,7 +412,7 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { let place_ty = place.ty(self.local_decls, self.tcx); if let ty::Array(_, len) = place_ty.ty.kind() { Const::Ty(self.tcx.types.usize, *len) - .try_eval_scalar(self.tcx, self.param_env) + .try_eval_scalar(self.tcx, self.typing_env) .map_or(FlatSet::Top, FlatSet::Elem) } else if let [ProjectionElem::Deref] = place.projection[..] { state.get_len(place.local.into(), &self.map) @@ -420,7 +421,7 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { } } Rvalue::Cast(CastKind::IntToInt | CastKind::IntToFloat, operand, ty) => { - let Ok(layout) = self.tcx.layout_of(self.param_env.and(*ty)) else { + let Ok(layout) = self.tcx.layout_of(self.typing_env.as_query_input(*ty)) else { return ValueOrPlace::Value(FlatSet::Top); }; match self.eval_operand(operand, state) { @@ -434,7 +435,7 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { } } Rvalue::Cast(CastKind::FloatToInt | CastKind::FloatToFloat, operand, ty) => { - let Ok(layout) = self.tcx.layout_of(self.param_env.and(*ty)) else { + let Ok(layout) = self.tcx.layout_of(self.typing_env.as_query_input(*ty)) else { return ValueOrPlace::Value(FlatSet::Top); }; match self.eval_operand(operand, state) { @@ -470,7 +471,7 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { FlatSet::Top => FlatSet::Top, }, Rvalue::NullaryOp(null_op, ty) => { - let Ok(layout) = self.tcx.layout_of(self.param_env.and(*ty)) else { + let Ok(layout) = self.tcx.layout_of(self.typing_env.as_query_input(*ty)) else { return ValueOrPlace::Value(FlatSet::Top); }; let val = match null_op { @@ -479,7 +480,7 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { NullOp::OffsetOf(fields) => self .ecx .tcx - .offset_of_subfield(self.ecx.param_env(), layout, fields.iter()) + .offset_of_subfield(self.typing_env, layout, fields.iter()) .bytes(), _ => return ValueOrPlace::Value(FlatSet::Top), }; @@ -514,7 +515,7 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { ) -> FlatSet<Scalar> { constant .const_ - .try_eval_scalar(self.tcx, self.param_env) + .try_eval_scalar(self.tcx, self.typing_env) .map_or(FlatSet::Top, FlatSet::Elem) } @@ -533,8 +534,13 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { // This allows the set of visited edges to grow monotonically with the lattice. FlatSet::Bottom => TerminatorEdges::None, FlatSet::Elem(scalar) => { - let choice = scalar.assert_scalar_int().to_bits_unchecked(); - TerminatorEdges::Single(targets.target_for_value(choice)) + if let Ok(scalar_int) = scalar.try_to_scalar_int() { + TerminatorEdges::Single( + targets.target_for_value(scalar_int.to_bits_unchecked()), + ) + } else { + TerminatorEdges::SwitchInt { discr, targets } + } } FlatSet::Top => TerminatorEdges::SwitchInt { discr, targets }, } @@ -554,7 +560,8 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { } else if rhs.projection.first() == Some(&PlaceElem::Deref) && let FlatSet::Elem(pointer) = state.get(rhs.local.into(), &self.map) && let rhs_ty = self.local_decls[rhs.local].ty - && let Ok(rhs_layout) = self.tcx.layout_of(self.param_env.and(rhs_ty)) + && let Ok(rhs_layout) = + self.tcx.layout_of(self.typing_env.as_query_input(rhs_ty)) { let op = ImmTy::from_scalar(pointer, rhs_layout).into(); self.assign_constant(state, place, op, rhs.projection); @@ -614,8 +621,10 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { TrackElem::DerefLen => { let op: OpTy<'_> = self.ecx.deref_pointer(op).discard_err()?.into(); let len_usize = op.len(&self.ecx).discard_err()?; - let layout = - self.tcx.layout_of(self.param_env.and(self.tcx.types.usize)).unwrap(); + let layout = self + .tcx + .layout_of(self.typing_env.as_query_input(self.tcx.types.usize)) + .unwrap(); Some(ImmTy::from_uint(len_usize, layout).into()) } }, @@ -702,9 +711,11 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { FlatSet::Top => FlatSet::Top, FlatSet::Elem(scalar) => { let ty = op.ty(self.local_decls, self.tcx); - self.tcx.layout_of(self.param_env.and(ty)).map_or(FlatSet::Top, |layout| { - FlatSet::Elem(ImmTy::from_scalar(scalar, layout)) - }) + self.tcx + .layout_of(self.typing_env.as_query_input(ty)) + .map_or(FlatSet::Top, |layout| { + FlatSet::Elem(ImmTy::from_scalar(scalar, layout)) + }) } FlatSet::Bottom => FlatSet::Bottom, } @@ -714,7 +725,7 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { if !enum_ty.is_enum() { return None; } - let enum_ty_layout = self.tcx.layout_of(self.param_env.and(enum_ty)).ok()?; + let enum_ty_layout = self.tcx.layout_of(self.typing_env.as_query_input(enum_ty)).ok()?; let discr_value = self.ecx.discriminant_for_variant(enum_ty_layout.ty, variant_index).discard_err()?; Some(discr_value.to_scalar()) @@ -941,16 +952,12 @@ fn try_write_constant<'tcx>( interp_ok(()) } -impl<'mir, 'tcx> ResultsVisitor<'mir, 'tcx, Results<'tcx, ConstAnalysis<'_, 'tcx>>> - for Collector<'_, 'tcx> -{ - type Domain = State<FlatSet<Scalar>>; - +impl<'mir, 'tcx> ResultsVisitor<'mir, 'tcx, ConstAnalysis<'_, 'tcx>> for Collector<'_, 'tcx> { #[instrument(level = "trace", skip(self, results, statement))] - fn visit_statement_before_primary_effect( + fn visit_after_early_statement_effect( &mut self, results: &mut Results<'tcx, ConstAnalysis<'_, 'tcx>>, - state: &Self::Domain, + state: &State<FlatSet<Scalar>>, statement: &'mir Statement<'tcx>, location: Location, ) { @@ -969,10 +976,10 @@ impl<'mir, 'tcx> ResultsVisitor<'mir, 'tcx, Results<'tcx, ConstAnalysis<'_, 'tcx } #[instrument(level = "trace", skip(self, results, statement))] - fn visit_statement_after_primary_effect( + fn visit_after_primary_statement_effect( &mut self, results: &mut Results<'tcx, ConstAnalysis<'_, 'tcx>>, - state: &Self::Domain, + state: &State<FlatSet<Scalar>>, statement: &'mir Statement<'tcx>, location: Location, ) { @@ -994,10 +1001,10 @@ impl<'mir, 'tcx> ResultsVisitor<'mir, 'tcx, Results<'tcx, ConstAnalysis<'_, 'tcx } } - fn visit_terminator_before_primary_effect( + fn visit_after_early_terminator_effect( &mut self, results: &mut Results<'tcx, ConstAnalysis<'_, 'tcx>>, - state: &Self::Domain, + state: &State<FlatSet<Scalar>>, terminator: &'mir Terminator<'tcx>, location: Location, ) { diff --git a/compiler/rustc_mir_transform/src/dead_store_elimination.rs b/compiler/rustc_mir_transform/src/dead_store_elimination.rs index 2898f82e25c..0c75cdadc92 100644 --- a/compiler/rustc_mir_transform/src/dead_store_elimination.rs +++ b/compiler/rustc_mir_transform/src/dead_store_elimination.rs @@ -99,7 +99,8 @@ fn eliminate<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { | StatementKind::Intrinsic(_) | StatementKind::ConstEvalCounter | StatementKind::PlaceMention(_) - | StatementKind::Nop => (), + | StatementKind::BackwardIncompatibleDropHint { .. } + | StatementKind::Nop => {} StatementKind::FakeRead(_) | StatementKind::AscribeUserType(_, _) => { bug!("{:?} not found in this MIR phase!", statement.kind) diff --git a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs index 753bae8e156..67b215c7c9d 100644 --- a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs +++ b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs @@ -198,7 +198,7 @@ pub(super) fn deduced_param_attrs<'tcx>( // see [1]. // // [1]: https://github.com/rust-lang/rust/pull/103172#discussion_r999139997 - let param_env = tcx.param_env_reveal_all_normalized(def_id); + let typing_env = body.typing_env(tcx); let mut deduced_param_attrs = tcx.arena.alloc_from_iter( body.local_decls.iter().skip(1).take(body.arg_count).enumerate().map( |(arg_index, local_decl)| DeducedParamAttrs { @@ -207,8 +207,8 @@ pub(super) fn deduced_param_attrs<'tcx>( // their generic parameters, otherwise we'll see exponential // blow-up in compile times: #113372 && tcx - .normalize_erasing_regions(param_env, local_decl.ty) - .is_freeze(tcx, param_env), + .normalize_erasing_regions(typing_env, local_decl.ty) + .is_freeze(tcx, typing_env), }, ), ); diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index beeab0d4a66..8f977d2979e 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -217,11 +217,6 @@ impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation { else { continue; }; - if !tcx.consider_optimizing(|| { - format!("{} round {}", tcx.def_path_str(def_id), round_count) - }) { - break; - } // Replace `src` by `dest` everywhere. merges.insert(*src, *dest); @@ -581,7 +576,7 @@ impl WriteInfo { | Rvalue::RawPtr(_, _) | Rvalue::Len(_) | Rvalue::Discriminant(_) - | Rvalue::CopyForDeref(_) => (), + | Rvalue::CopyForDeref(_) => {} } } // Retags are technically also reads, but reporting them as a write suffices @@ -596,7 +591,8 @@ impl WriteInfo { | StatementKind::Coverage(_) | StatementKind::StorageLive(_) | StatementKind::StorageDead(_) - | StatementKind::PlaceMention(_) => (), + | StatementKind::BackwardIncompatibleDropHint { .. } + | StatementKind::PlaceMention(_) => {} StatementKind::FakeRead(_) | StatementKind::AscribeUserType(_, _) => { bug!("{:?} not found in this MIR phase", statement) } diff --git a/compiler/rustc_mir_transform/src/early_otherwise_branch.rs b/compiler/rustc_mir_transform/src/early_otherwise_branch.rs index 704ed508b22..91e1395e764 100644 --- a/compiler/rustc_mir_transform/src/early_otherwise_branch.rs +++ b/compiler/rustc_mir_transform/src/early_otherwise_branch.rs @@ -108,10 +108,6 @@ impl<'tcx> crate::MirPass<'tcx> for EarlyOtherwiseBranch { let parent = BasicBlock::from_usize(i); let Some(opt_data) = evaluate_candidate(tcx, body, parent) else { continue }; - if !tcx.consider_optimizing(|| format!("EarlyOtherwiseBranch {opt_data:?}")) { - break; - } - trace!("SUCCESS: found optimization possibility to apply: {opt_data:?}"); should_cleanup = true; @@ -133,18 +129,29 @@ impl<'tcx> crate::MirPass<'tcx> for EarlyOtherwiseBranch { let mut patch = MirPatch::new(body); - // create temp to store second discriminant in, `_s` in example above - let second_discriminant_temp = - patch.new_temp(opt_data.child_ty, opt_data.child_source.span); + let (second_discriminant_temp, second_operand) = if opt_data.need_hoist_discriminant { + // create temp to store second discriminant in, `_s` in example above + let second_discriminant_temp = + patch.new_temp(opt_data.child_ty, opt_data.child_source.span); - patch.add_statement(parent_end, StatementKind::StorageLive(second_discriminant_temp)); + patch.add_statement( + parent_end, + StatementKind::StorageLive(second_discriminant_temp), + ); - // create assignment of discriminant - patch.add_assign( - parent_end, - Place::from(second_discriminant_temp), - Rvalue::Discriminant(opt_data.child_place), - ); + // create assignment of discriminant + patch.add_assign( + parent_end, + Place::from(second_discriminant_temp), + Rvalue::Discriminant(opt_data.child_place), + ); + ( + Some(second_discriminant_temp), + Operand::Move(Place::from(second_discriminant_temp)), + ) + } else { + (None, Operand::Copy(opt_data.child_place)) + }; // create temp to store inequality comparison between the two discriminants, `_t` in // example above @@ -153,11 +160,9 @@ impl<'tcx> crate::MirPass<'tcx> for EarlyOtherwiseBranch { let comp_temp = patch.new_temp(comp_res_type, opt_data.child_source.span); patch.add_statement(parent_end, StatementKind::StorageLive(comp_temp)); - // create inequality comparison between the two discriminants - let comp_rvalue = Rvalue::BinaryOp( - nequal, - Box::new((parent_op.clone(), Operand::Move(Place::from(second_discriminant_temp)))), - ); + // create inequality comparison + let comp_rvalue = + Rvalue::BinaryOp(nequal, Box::new((parent_op.clone(), second_operand))); patch.add_statement( parent_end, StatementKind::Assign(Box::new((Place::from(comp_temp), comp_rvalue))), @@ -174,14 +179,17 @@ impl<'tcx> crate::MirPass<'tcx> for EarlyOtherwiseBranch { let eq_targets = SwitchTargets::new(eq_new_targets, parent_targets.otherwise()); // Create `bbEq` in example above - let eq_switch = BasicBlockData::new(Some(Terminator { - source_info: bbs[parent].terminator().source_info, - kind: TerminatorKind::SwitchInt { - // switch on the first discriminant, so we can mark the second one as dead - discr: parent_op, - targets: eq_targets, - }, - })); + let eq_switch = BasicBlockData::new( + Some(Terminator { + source_info: bbs[parent].terminator().source_info, + kind: TerminatorKind::SwitchInt { + // switch on the first discriminant, so we can mark the second one as dead + discr: parent_op, + targets: eq_targets, + }, + }), + bbs[parent].is_cleanup, + ); let eq_bb = patch.new_block(eq_switch); @@ -193,8 +201,13 @@ impl<'tcx> crate::MirPass<'tcx> for EarlyOtherwiseBranch { TerminatorKind::if_(Operand::Move(Place::from(comp_temp)), true_case, false_case), ); - // generate StorageDead for the second_discriminant_temp not in use anymore - patch.add_statement(parent_end, StatementKind::StorageDead(second_discriminant_temp)); + if let Some(second_discriminant_temp) = second_discriminant_temp { + // generate StorageDead for the second_discriminant_temp not in use anymore + patch.add_statement( + parent_end, + StatementKind::StorageDead(second_discriminant_temp), + ); + } // Generate a StorageDead for comp_temp in each of the targets, since we moved it into // the switch @@ -222,6 +235,7 @@ struct OptimizationData<'tcx> { child_place: Place<'tcx>, child_ty: Ty<'tcx>, child_source: SourceInfo, + need_hoist_discriminant: bool, } fn evaluate_candidate<'tcx>( @@ -230,49 +244,21 @@ fn evaluate_candidate<'tcx>( parent: BasicBlock, ) -> Option<OptimizationData<'tcx>> { let bbs = &body.basic_blocks; + // NB: If this BB is a cleanup, we may need to figure out what else needs to be handled. + if bbs[parent].is_cleanup { + return None; + } let TerminatorKind::SwitchInt { targets, discr: parent_discr } = &bbs[parent].terminator().kind else { return None; }; let parent_ty = parent_discr.ty(body.local_decls(), tcx); - if !bbs[targets.otherwise()].is_empty_unreachable() { - // Someone could write code like this: - // ```rust - // let Q = val; - // if discriminant(P) == otherwise { - // let ptr = &mut Q as *mut _ as *mut u8; - // // It may be difficult for us to effectively determine whether values are valid. - // // Invalid values can come from all sorts of corners. - // unsafe { *ptr = 10; } - // } - // - // match P { - // A => match Q { - // A => { - // // code - // } - // _ => { - // // don't use Q - // } - // } - // _ => { - // // don't use Q - // } - // }; - // ``` - // - // Hoisting the `discriminant(Q)` out of the `A` arm causes us to compute the discriminant - // of an invalid value, which is UB. - // In order to fix this, **we would either need to show that the discriminant computation of - // `place` is computed in all branches**. - // FIXME(#95162) For the moment, we adopt a conservative approach and - // consider only the `otherwise` branch has no statements and an unreachable terminator. - return None; - } let (_, child) = targets.iter().next()?; - let child_terminator = &bbs[child].terminator(); - let TerminatorKind::SwitchInt { targets: child_targets, discr: child_discr } = - &child_terminator.kind + + let Terminator { + kind: TerminatorKind::SwitchInt { targets: child_targets, discr: child_discr }, + source_info, + } = bbs[child].terminator() else { return None; }; @@ -280,25 +266,115 @@ fn evaluate_candidate<'tcx>( if child_ty != parent_ty { return None; } - let Some(StatementKind::Assign(boxed)) = &bbs[child].statements.first().map(|x| &x.kind) else { + + // We only handle: + // ``` + // bb4: { + // _8 = discriminant((_3.1: Enum1)); + // switchInt(move _8) -> [2: bb7, otherwise: bb1]; + // } + // ``` + // and + // ``` + // bb2: { + // switchInt((_3.1: u64)) -> [1: bb5, otherwise: bb1]; + // } + // ``` + if bbs[child].statements.len() > 1 { return None; + } + + // When thie BB has exactly one statement, this statement should be discriminant. + let need_hoist_discriminant = bbs[child].statements.len() == 1; + let child_place = if need_hoist_discriminant { + if !bbs[targets.otherwise()].is_empty_unreachable() { + // Someone could write code like this: + // ```rust + // let Q = val; + // if discriminant(P) == otherwise { + // let ptr = &mut Q as *mut _ as *mut u8; + // // It may be difficult for us to effectively determine whether values are valid. + // // Invalid values can come from all sorts of corners. + // unsafe { *ptr = 10; } + // } + // + // match P { + // A => match Q { + // A => { + // // code + // } + // _ => { + // // don't use Q + // } + // } + // _ => { + // // don't use Q + // } + // }; + // ``` + // + // Hoisting the `discriminant(Q)` out of the `A` arm causes us to compute the discriminant of an + // invalid value, which is UB. + // In order to fix this, **we would either need to show that the discriminant computation of + // `place` is computed in all branches**. + // FIXME(#95162) For the moment, we adopt a conservative approach and + // consider only the `otherwise` branch has no statements and an unreachable terminator. + return None; + } + // Handle: + // ``` + // bb4: { + // _8 = discriminant((_3.1: Enum1)); + // switchInt(move _8) -> [2: bb7, otherwise: bb1]; + // } + // ``` + let [ + Statement { + kind: StatementKind::Assign(box (_, Rvalue::Discriminant(child_place))), + .. + }, + ] = bbs[child].statements.as_slice() + else { + return None; + }; + *child_place + } else { + // Handle: + // ``` + // bb2: { + // switchInt((_3.1: u64)) -> [1: bb5, otherwise: bb1]; + // } + // ``` + let Operand::Copy(child_place) = child_discr else { + return None; + }; + *child_place }; - let (_, Rvalue::Discriminant(child_place)) = &**boxed else { - return None; + let destination = if need_hoist_discriminant || bbs[targets.otherwise()].is_empty_unreachable() + { + child_targets.otherwise() + } else { + targets.otherwise() }; - let destination = child_targets.otherwise(); // Verify that the optimization is legal for each branch for (value, child) in targets.iter() { - if !verify_candidate_branch(&bbs[child], value, *child_place, destination) { + if !verify_candidate_branch( + &bbs[child], + value, + child_place, + destination, + need_hoist_discriminant, + ) { return None; } } Some(OptimizationData { destination, - child_place: *child_place, + child_place, child_ty, - child_source: child_terminator.source_info, + child_source: *source_info, + need_hoist_discriminant, }) } @@ -307,31 +383,48 @@ fn verify_candidate_branch<'tcx>( value: u128, place: Place<'tcx>, destination: BasicBlock, + need_hoist_discriminant: bool, ) -> bool { - // In order for the optimization to be correct, the branch must... - // ...have exactly one statement - if let [statement] = branch.statements.as_slice() - // ...assign the discriminant of `place` in that statement - && let StatementKind::Assign(boxed) = &statement.kind - && let (discr_place, Rvalue::Discriminant(from_place)) = &**boxed - && *from_place == place - // ...make that assignment to a local - && discr_place.projection.is_empty() - // ...terminate on a `SwitchInt` that invalidates that local - && let TerminatorKind::SwitchInt { discr: switch_op, targets, .. } = - &branch.terminator().kind - && *switch_op == Operand::Move(*discr_place) - // ...fall through to `destination` if the switch misses - && destination == targets.otherwise() - // ...have a branch for value `value` - && let mut iter = targets.iter() - && let Some((target_value, _)) = iter.next() - && target_value == value - // ...and have no more branches - && iter.next().is_none() - { - true + // In order for the optimization to be correct, the terminator must be a `SwitchInt`. + let TerminatorKind::SwitchInt { discr: switch_op, targets } = &branch.terminator().kind else { + return false; + }; + if need_hoist_discriminant { + // If we need hoist discriminant, the branch must have exactly one statement. + let [statement] = branch.statements.as_slice() else { + return false; + }; + // The statement must assign the discriminant of `place`. + let StatementKind::Assign(box (discr_place, Rvalue::Discriminant(from_place))) = + statement.kind + else { + return false; + }; + if from_place != place { + return false; + } + // The assignment must invalidate a local that terminate on a `SwitchInt`. + if !discr_place.projection.is_empty() || *switch_op != Operand::Move(discr_place) { + return false; + } } else { - false + // If we don't need hoist discriminant, the branch must not have any statements. + if !branch.statements.is_empty() { + return false; + } + // The place on `SwitchInt` must be the same. + if *switch_op != Operand::Copy(place) { + return false; + } } + // It must fall through to `destination` if the switch misses. + if destination != targets.otherwise() { + return false; + } + // It must have exactly one branch for value `value` and have no more branches. + let mut iter = targets.iter(); + let (Some((target_value, _)), None) = (iter.next(), iter.next()) else { + return false; + }; + target_value == value } diff --git a/compiler/rustc_mir_transform/src/elaborate_drops.rs b/compiler/rustc_mir_transform/src/elaborate_drops.rs index 74572100db3..3ebc9113725 100644 --- a/compiler/rustc_mir_transform/src/elaborate_drops.rs +++ b/compiler/rustc_mir_transform/src/elaborate_drops.rs @@ -12,7 +12,7 @@ use rustc_mir_dataflow::elaborate_drops::{ use rustc_mir_dataflow::impls::{MaybeInitializedPlaces, MaybeUninitializedPlaces}; use rustc_mir_dataflow::move_paths::{LookupResult, MoveData, MovePathIndex}; use rustc_mir_dataflow::{ - Analysis, MoveDataParamEnv, ResultsCursor, on_all_children_bits, on_lookup_result_bits, + Analysis, MoveDataTypingEnv, ResultsCursor, on_all_children_bits, on_lookup_result_bits, }; use rustc_span::Span; use tracing::{debug, instrument}; @@ -53,14 +53,14 @@ impl<'tcx> crate::MirPass<'tcx> for ElaborateDrops { #[instrument(level = "trace", skip(self, tcx, body))] fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { debug!("elaborate_drops({:?} @ {:?})", body.source, body.span); - - let def_id = body.source.def_id(); - let param_env = tcx.param_env_reveal_all_normalized(def_id); + // FIXME(#132279): This is used during the phase transition from analysis + // to runtime, so we have to manually specify the correct typing mode. + let typing_env = ty::TypingEnv::post_analysis(tcx, body.source.def_id()); // For types that do not need dropping, the behaviour is trivial. So we only need to track // init/uninit for types that do need dropping. - let move_data = MoveData::gather_moves(body, tcx, |ty| ty.needs_drop(tcx, param_env)); + let move_data = MoveData::gather_moves(body, tcx, |ty| ty.needs_drop(tcx, typing_env)); let elaborate_patch = { - let env = MoveDataParamEnv { move_data, param_env }; + let env = MoveDataTypingEnv { move_data, typing_env }; let mut inits = MaybeInitializedPlaces::new(tcx, body, &env.move_data) .skipping_unreachable_unwind() @@ -127,7 +127,7 @@ impl InitializationData<'_, '_> { self.uninits.seek_before_primary_effect(loc); } - fn maybe_live_dead(&self, path: MovePathIndex) -> (bool, bool) { + fn maybe_init_uninit(&self, path: MovePathIndex) -> (bool, bool) { (self.inits.get().contains(path), self.uninits.get().contains(path)) } } @@ -147,29 +147,29 @@ impl<'a, 'tcx> DropElaborator<'a, 'tcx> for ElaborateDropsCtxt<'a, 'tcx> { self.tcx } - fn param_env(&self) -> ty::ParamEnv<'tcx> { - self.param_env() + fn typing_env(&self) -> ty::TypingEnv<'tcx> { + self.env.typing_env } #[instrument(level = "debug", skip(self), ret)] fn drop_style(&self, path: Self::Path, mode: DropFlagMode) -> DropStyle { - let ((maybe_live, maybe_dead), multipart) = match mode { - DropFlagMode::Shallow => (self.init_data.maybe_live_dead(path), false), + let ((maybe_init, maybe_uninit), multipart) = match mode { + DropFlagMode::Shallow => (self.init_data.maybe_init_uninit(path), false), DropFlagMode::Deep => { - let mut some_live = false; - let mut some_dead = false; + let mut some_maybe_init = false; + let mut some_maybe_uninit = false; let mut children_count = 0; on_all_children_bits(self.move_data(), path, |child| { - let (live, dead) = self.init_data.maybe_live_dead(child); - debug!("elaborate_drop: state({:?}) = {:?}", child, (live, dead)); - some_live |= live; - some_dead |= dead; + let (maybe_init, maybe_uninit) = self.init_data.maybe_init_uninit(child); + debug!("elaborate_drop: state({:?}) = {:?}", child, (maybe_init, maybe_uninit)); + some_maybe_init |= maybe_init; + some_maybe_uninit |= maybe_uninit; children_count += 1; }); - ((some_live, some_dead), children_count != 1) + ((some_maybe_init, some_maybe_uninit), children_count != 1) } }; - match (maybe_live, maybe_dead, multipart) { + match (maybe_init, maybe_uninit, multipart) { (false, _, _) => DropStyle::Dead, (true, false, _) => DropStyle::Static, (true, true, false) => DropStyle::Conditional, @@ -229,7 +229,7 @@ impl<'a, 'tcx> DropElaborator<'a, 'tcx> for ElaborateDropsCtxt<'a, 'tcx> { struct ElaborateDropsCtxt<'a, 'tcx> { tcx: TyCtxt<'tcx>, body: &'a Body<'tcx>, - env: &'a MoveDataParamEnv<'tcx>, + env: &'a MoveDataTypingEnv<'tcx>, init_data: InitializationData<'a, 'tcx>, drop_flags: IndexVec<MovePathIndex, Option<Local>>, patch: MirPatch<'tcx>, @@ -246,10 +246,6 @@ impl<'a, 'tcx> ElaborateDropsCtxt<'a, 'tcx> { &self.env.move_data } - fn param_env(&self) -> ty::ParamEnv<'tcx> { - self.env.param_env - } - fn create_drop_flag(&mut self, index: MovePathIndex, span: Span) { let patch = &mut self.patch; debug!("create_drop_flag({:?})", self.body.span); @@ -287,15 +283,15 @@ impl<'a, 'tcx> ElaborateDropsCtxt<'a, 'tcx> { LookupResult::Exact(path) => { self.init_data.seek_before(self.body.terminator_loc(bb)); on_all_children_bits(self.move_data(), path, |child| { - let (maybe_live, maybe_dead) = self.init_data.maybe_live_dead(child); + let (maybe_init, maybe_uninit) = self.init_data.maybe_init_uninit(child); debug!( "collect_drop_flags: collecting {:?} from {:?}@{:?} - {:?}", child, place, path, - (maybe_live, maybe_dead) + (maybe_init, maybe_uninit) ); - if maybe_live && maybe_dead { + if maybe_init && maybe_uninit { self.create_drop_flag(child, terminator.source_info.span) } }); @@ -307,8 +303,8 @@ impl<'a, 'tcx> ElaborateDropsCtxt<'a, 'tcx> { } self.init_data.seek_before(self.body.terminator_loc(bb)); - let (_maybe_live, maybe_dead) = self.init_data.maybe_live_dead(parent); - if maybe_dead { + let (_maybe_init, maybe_uninit) = self.init_data.maybe_init_uninit(parent); + if maybe_uninit { self.tcx.dcx().span_delayed_bug( terminator.source_info.span, format!( @@ -335,7 +331,7 @@ impl<'a, 'tcx> ElaborateDropsCtxt<'a, 'tcx> { if !place .ty(&self.body.local_decls, self.tcx) .ty - .needs_drop(self.tcx, self.env.param_env) + .needs_drop(self.tcx, self.typing_env()) { self.patch.patch_terminator(bb, TerminatorKind::Goto { target }); continue; diff --git a/compiler/rustc_mir_transform/src/errors.rs b/compiler/rustc_mir_transform/src/errors.rs index 8b309147c64..2d9eeddea2e 100644 --- a/compiler/rustc_mir_transform/src/errors.rs +++ b/compiler/rustc_mir_transform/src/errors.rs @@ -38,6 +38,12 @@ pub(crate) struct UnalignedPackedRef { pub span: Span, } +#[derive(Diagnostic)] +#[diag(mir_transform_unknown_pass_name)] +pub(crate) struct UnknownPassName<'a> { + pub(crate) name: &'a str, +} + pub(crate) struct AssertLint<P> { pub span: Span, pub assert_kind: AssertKind<P>, diff --git a/compiler/rustc_mir_transform/src/function_item_references.rs b/compiler/rustc_mir_transform/src/function_item_references.rs index 2945fc6f3d5..fb21bf9977f 100644 --- a/compiler/rustc_mir_transform/src/function_item_references.rs +++ b/compiler/rustc_mir_transform/src/function_item_references.rs @@ -5,9 +5,8 @@ use rustc_middle::mir::visit::Visitor; use rustc_middle::mir::*; use rustc_middle::ty::{self, EarlyBinder, GenericArgsRef, Ty, TyCtxt}; use rustc_session::lint::builtin::FUNCTION_ITEM_REFERENCES; -use rustc_span::Span; use rustc_span::source_map::Spanned; -use rustc_span::symbol::sym; +use rustc_span::{Span, sym}; use crate::errors; diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs index 274eea9563f..d5a813ec8ec 100644 --- a/compiler/rustc_mir_transform/src/gvn.rs +++ b/compiler/rustc_mir_transform/src/gvn.rs @@ -100,7 +100,7 @@ use rustc_middle::bug; use rustc_middle::mir::interpret::GlobalAlloc; use rustc_middle::mir::visit::*; use rustc_middle::mir::*; -use rustc_middle::ty::layout::{HasParamEnv, LayoutOf}; +use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::DUMMY_SP; use rustc_span::def_id::DefId; @@ -120,12 +120,12 @@ impl<'tcx> crate::MirPass<'tcx> for GVN { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { debug!(def_id = ?body.source.def_id()); - let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); - let ssa = SsaLocals::new(tcx, body, param_env); + let typing_env = body.typing_env(tcx); + let ssa = SsaLocals::new(tcx, body, typing_env); // Clone dominators because we need them while mutating the body. let dominators = body.basic_blocks.dominators().clone(); - let mut state = VnState::new(tcx, body, param_env, &ssa, dominators, &body.local_decls); + let mut state = VnState::new(tcx, body, typing_env, &ssa, dominators, &body.local_decls); ssa.for_each_assignment_mut( body.basic_blocks.as_mut_preserves_cfg(), |local, value, location| { @@ -241,7 +241,6 @@ enum Value<'tcx> { struct VnState<'body, 'tcx> { tcx: TyCtxt<'tcx>, ecx: InterpCx<'tcx, DummyMachine>, - param_env: ty::ParamEnv<'tcx>, local_decls: &'body LocalDecls<'tcx>, /// Value stored in each local. locals: IndexVec<Local, Option<VnIndex>>, @@ -266,7 +265,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { fn new( tcx: TyCtxt<'tcx>, body: &Body<'tcx>, - param_env: ty::ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, ssa: &'body SsaLocals, dominators: Dominators<BasicBlock>, local_decls: &'body LocalDecls<'tcx>, @@ -280,8 +279,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { + 4 * body.basic_blocks.len(); VnState { tcx, - ecx: InterpCx::new(tcx, DUMMY_SP, param_env, DummyMachine), - param_env, + ecx: InterpCx::new(tcx, DUMMY_SP, typing_env, DummyMachine), local_decls, locals: IndexVec::from_elem(None, local_decls), rev_locals: IndexVec::with_capacity(num_values), @@ -295,6 +293,10 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { } } + fn typing_env(&self) -> ty::TypingEnv<'tcx> { + self.ecx.typing_env() + } + #[instrument(level = "trace", skip(self), ret)] fn insert(&mut self, value: Value<'tcx>) -> VnIndex { let (index, new) = self.values.insert_full(value); @@ -343,7 +345,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { // Only register the value if its type is `Sized`, as we will emit copies of it. let is_sized = !self.feature_unsized_locals - || self.local_decls[local].ty.is_sized(self.tcx, self.param_env); + || self.local_decls[local].ty.is_sized(self.tcx, self.typing_env()); if is_sized { self.rev_locals[value].push(local); } @@ -531,7 +533,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { NullOp::OffsetOf(fields) => self .ecx .tcx - .offset_of_subfield(self.ecx.param_env(), layout, fields.iter()) + .offset_of_subfield(self.typing_env(), layout, fields.iter()) .bytes(), NullOp::UbChecks => return None, }; @@ -636,9 +638,11 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { let proj = match proj { ProjectionElem::Deref => { let ty = place.ty(self.local_decls, self.tcx).ty; - if let Some(Mutability::Not) = ty.ref_mutability() + // unsound: https://github.com/rust-lang/rust/issues/130853 + if self.tcx.sess.opts.unstable_opts.unsound_mir_opts + && let Some(Mutability::Not) = ty.ref_mutability() && let Some(pointee_ty) = ty.builtin_deref(true) - && pointee_ty.is_freeze(self.tcx, self.param_env) + && pointee_ty.is_freeze(self.tcx, self.typing_env()) { // An immutable borrow `_x` always points to the same value for the // lifetime of the borrow, so we can merge all instances of `*_x`. @@ -1057,7 +1061,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { && let ty::RawPtr(from_pointee_ty, from_mtbl) = cast_from.kind() && let ty::RawPtr(_, output_mtbl) = output_pointer_ty.kind() && from_mtbl == output_mtbl - && from_pointee_ty.is_sized(self.tcx, self.param_env) + && from_pointee_ty.is_sized(self.tcx, self.typing_env()) { fields[0] = *cast_value; *data_pointer_ty = *cast_from; @@ -1379,7 +1383,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { && let Value::Aggregate(AggregateTy::RawPtr { data_pointer_ty, .. }, _, fields) = self.get(value) && let ty::RawPtr(to_pointee, _) = to.kind() - && to_pointee.is_sized(self.tcx, self.param_env) + && to_pointee.is_sized(self.tcx, self.typing_env()) { from = *data_pointer_ty; value = fields[0]; @@ -1476,8 +1480,9 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { if left_meta_ty == right_meta_ty { true } else if let Ok(left) = - self.tcx.try_normalize_erasing_regions(self.param_env, left_meta_ty) - && let Ok(right) = self.tcx.try_normalize_erasing_regions(self.param_env, right_meta_ty) + self.tcx.try_normalize_erasing_regions(self.typing_env(), left_meta_ty) + && let Ok(right) = + self.tcx.try_normalize_erasing_regions(self.typing_env(), right_meta_ty) { left == right } else { diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index e95ab4ffe16..35699acb318 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -4,7 +4,7 @@ use std::iter; use std::ops::{Range, RangeFrom}; use rustc_abi::{ExternAbi, FieldIdx}; -use rustc_attr::InlineAttr; +use rustc_attr_parsing::InlineAttr; use rustc_hir::def::DefKind; use rustc_hir::def_id::DefId; use rustc_index::Idx; @@ -13,9 +13,7 @@ use rustc_middle::bug; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::mir::visit::*; use rustc_middle::mir::*; -use rustc_middle::ty::{ - self, Instance, InstanceKind, ParamEnv, Ty, TyCtxt, TypeFlags, TypeVisitableExt, -}; +use rustc_middle::ty::{self, Instance, InstanceKind, Ty, TyCtxt, TypeFlags, TypeVisitableExt}; use rustc_session::config::{DebugInfo, OptLevel}; use rustc_span::source_map::Spanned; use rustc_span::sym; @@ -94,12 +92,12 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { return false; } - let param_env = tcx.param_env_reveal_all_normalized(def_id); + let typing_env = body.typing_env(tcx); let codegen_fn_attrs = tcx.codegen_fn_attrs(def_id); let mut this = Inliner { tcx, - param_env, + typing_env, codegen_fn_attrs, history: Vec::new(), changed: false, @@ -115,7 +113,7 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { struct Inliner<'tcx> { tcx: TyCtxt<'tcx>, - param_env: ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, /// Caller codegen attributes. codegen_fn_attrs: &'tcx CodegenFnAttrs, /// Stack of inlined instances. @@ -201,7 +199,8 @@ impl<'tcx> Inliner<'tcx> { let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() }; let destination_ty = destination.ty(&caller_body.local_decls, self.tcx).ty; for arg in args { - if !arg.node.ty(&caller_body.local_decls, self.tcx).is_sized(self.tcx, self.param_env) { + if !arg.node.ty(&caller_body.local_decls, self.tcx).is_sized(self.tcx, self.typing_env) + { // We do not allow inlining functions with unsized params. Inlining these functions // could create unsized locals, which are unsound and being phased out. return Err("Call has unsized argument"); @@ -211,15 +210,9 @@ impl<'tcx> Inliner<'tcx> { let callee_body = try_instance_mir(self.tcx, callsite.callee.def)?; self.check_mir_body(callsite, callee_body, callee_attrs, cross_crate_inlinable)?; - if !self.tcx.consider_optimizing(|| { - format!("Inline {:?} into {:?}", callsite.callee, caller_body.source) - }) { - return Err("optimization fuel exhausted"); - } - let Ok(callee_body) = callsite.callee.try_instantiate_mir_and_normalize_erasing_regions( self.tcx, - self.param_env, + self.typing_env, ty::EarlyBinder::bind(callee_body.clone()), ) else { return Err("failed to normalize callee body"); @@ -227,15 +220,7 @@ impl<'tcx> Inliner<'tcx> { // Normally, this shouldn't be required, but trait normalization failure can create a // validation ICE. - if !validate_types( - self.tcx, - MirPhase::Runtime(RuntimePhase::Optimized), - self.param_env, - &callee_body, - &caller_body, - ) - .is_empty() - { + if !validate_types(self.tcx, self.typing_env, &callee_body, &caller_body).is_empty() { return Err("failed to validate callee body"); } @@ -243,13 +228,7 @@ impl<'tcx> Inliner<'tcx> { // Normally, this shouldn't be required, but trait normalization failure can create a // validation ICE. let output_type = callee_body.return_ty(); - if !util::sub_types( - self.tcx, - caller_body.typing_mode(self.tcx), - self.param_env, - output_type, - destination_ty, - ) { + if !util::sub_types(self.tcx, self.typing_env, output_type, destination_ty) { trace!(?output_type, ?destination_ty); return Err("failed to normalize return type"); } @@ -279,13 +258,7 @@ impl<'tcx> Inliner<'tcx> { self_arg_ty.into_iter().chain(arg_tuple_tys).zip(callee_body.args_iter()) { let input_type = callee_body.local_decls[input].ty; - if !util::sub_types( - self.tcx, - caller_body.typing_mode(self.tcx), - self.param_env, - input_type, - arg_ty, - ) { + if !util::sub_types(self.tcx, self.typing_env, input_type, arg_ty) { trace!(?arg_ty, ?input_type); return Err("failed to normalize tuple argument type"); } @@ -294,13 +267,7 @@ impl<'tcx> Inliner<'tcx> { for (arg, input) in args.iter().zip(callee_body.args_iter()) { let input_type = callee_body.local_decls[input].ty; let arg_ty = arg.node.ty(&caller_body.local_decls, self.tcx); - if !util::sub_types( - self.tcx, - caller_body.typing_mode(self.tcx), - self.param_env, - input_type, - arg_ty, - ) { + if !util::sub_types(self.tcx, self.typing_env, input_type, arg_ty) { trace!(?arg_ty, ?input_type); return Err("failed to normalize argument type"); } @@ -402,9 +369,10 @@ impl<'tcx> Inliner<'tcx> { let func_ty = func.ty(caller_body, self.tcx); if let ty::FnDef(def_id, args) = *func_ty.kind() { // To resolve an instance its args have to be fully normalized. - let args = self.tcx.try_normalize_erasing_regions(self.param_env, args).ok()?; - let callee = - Instance::try_resolve(self.tcx, self.param_env, def_id, args).ok().flatten()?; + let args = self.tcx.try_normalize_erasing_regions(self.typing_env, args).ok()?; + let callee = Instance::try_resolve(self.tcx, self.typing_env, def_id, args) + .ok() + .flatten()?; if let InstanceKind::Virtual(..) | InstanceKind::Intrinsic(_) = callee.def { return None; @@ -528,7 +496,7 @@ impl<'tcx> Inliner<'tcx> { // FIXME: Give a bonus to functions with only a single caller let mut checker = - CostChecker::new(self.tcx, self.param_env, Some(callsite.callee), callee_body); + CostChecker::new(self.tcx, self.typing_env, Some(callsite.callee), callee_body); checker.add_function_level_costs(); @@ -552,7 +520,7 @@ impl<'tcx> Inliner<'tcx> { self.tcx, ty::EarlyBinder::bind(&place.ty(callee_body, tcx).ty), ); - if ty.needs_drop(tcx, self.param_env) + if ty.needs_drop(tcx, self.typing_env) && let UnwindAction::Cleanup(unwind) = unwind { work_list.push(unwind); @@ -604,11 +572,13 @@ impl<'tcx> Inliner<'tcx> { let return_block = if let Some(block) = target { // Prepare a new block for code that should execute when call returns. We don't use // target block directly since it might have other predecessors. - let mut data = BasicBlockData::new(Some(Terminator { - source_info: terminator.source_info, - kind: TerminatorKind::Goto { target: block }, - })); - data.is_cleanup = caller_body[block].is_cleanup; + let data = BasicBlockData::new( + Some(Terminator { + source_info: terminator.source_info, + kind: TerminatorKind::Goto { target: block }, + }), + caller_body[block].is_cleanup, + ); Some(caller_body.basic_blocks_mut().push(data)) } else { None diff --git a/compiler/rustc_mir_transform/src/inline/cycle.rs b/compiler/rustc_mir_transform/src/inline/cycle.rs index 9828e90de88..a40768300f5 100644 --- a/compiler/rustc_mir_transform/src/inline/cycle.rs +++ b/compiler/rustc_mir_transform/src/inline/cycle.rs @@ -15,7 +15,6 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( (root, target): (ty::Instance<'tcx>, LocalDefId), ) -> bool { trace!(%root, target = %tcx.def_path_str(target)); - let param_env = tcx.param_env_reveal_all_normalized(target); assert_ne!( root.def_id().expect_local(), target, @@ -31,11 +30,11 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( ); #[instrument( level = "debug", - skip(tcx, param_env, target, stack, seen, recursion_limiter, caller, recursion_limit) + skip(tcx, typing_env, target, stack, seen, recursion_limiter, caller, recursion_limit) )] fn process<'tcx>( tcx: TyCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, caller: ty::Instance<'tcx>, target: LocalDefId, stack: &mut Vec<ty::Instance<'tcx>>, @@ -47,13 +46,13 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( for &(callee, args) in tcx.mir_inliner_callees(caller.def) { let Ok(args) = caller.try_instantiate_mir_and_normalize_erasing_regions( tcx, - param_env, + typing_env, ty::EarlyBinder::bind(args), ) else { - trace!(?caller, ?param_env, ?args, "cannot normalize, skipping"); + trace!(?caller, ?typing_env, ?args, "cannot normalize, skipping"); continue; }; - let Ok(Some(callee)) = ty::Instance::try_resolve(tcx, param_env, callee, args) else { + let Ok(Some(callee)) = ty::Instance::try_resolve(tcx, typing_env, callee, args) else { trace!(?callee, "cannot resolve, skipping"); continue; }; @@ -115,7 +114,7 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( let found_recursion = ensure_sufficient_stack(|| { process( tcx, - param_env, + typing_env, callee, target, stack, @@ -146,7 +145,7 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( let recursion_limit = tcx.recursion_limit() / 2; process( tcx, - param_env, + ty::TypingEnv::post_analysis(tcx, target), root, target, &mut Vec::new(), diff --git a/compiler/rustc_mir_transform/src/instsimplify.rs b/compiler/rustc_mir_transform/src/instsimplify.rs index 9471c1b2a9a..1a65affe812 100644 --- a/compiler/rustc_mir_transform/src/instsimplify.rs +++ b/compiler/rustc_mir_transform/src/instsimplify.rs @@ -6,9 +6,8 @@ use rustc_hir::LangItem; use rustc_middle::bug; use rustc_middle::mir::*; use rustc_middle::ty::layout::ValidityRequirement; -use rustc_middle::ty::{self, GenericArgsRef, ParamEnv, Ty, TyCtxt, layout}; -use rustc_span::sym; -use rustc_span::symbol::Symbol; +use rustc_middle::ty::{self, GenericArgsRef, Ty, TyCtxt, layout}; +use rustc_span::{DUMMY_SP, Symbol, sym}; use crate::simplify::simplify_duplicate_switch_targets; use crate::take_array; @@ -34,7 +33,7 @@ impl<'tcx> crate::MirPass<'tcx> for InstSimplify { let ctx = InstSimplifyContext { tcx, local_decls: &body.local_decls, - param_env: tcx.param_env_reveal_all_normalized(body.source.def_id()), + typing_env: body.typing_env(tcx), }; let preserve_ub_checks = attr::contains_name(tcx.hir().krate_attrs(), sym::rustc_preserve_ub_checks); @@ -43,12 +42,11 @@ impl<'tcx> crate::MirPass<'tcx> for InstSimplify { match statement.kind { StatementKind::Assign(box (_place, ref mut rvalue)) => { if !preserve_ub_checks { - ctx.simplify_ub_check(&statement.source_info, rvalue); + ctx.simplify_ub_check(rvalue); } - ctx.simplify_bool_cmp(&statement.source_info, rvalue); - ctx.simplify_ref_deref(&statement.source_info, rvalue); - ctx.simplify_len(&statement.source_info, rvalue); - ctx.simplify_ptr_aggregate(&statement.source_info, rvalue); + ctx.simplify_bool_cmp(rvalue); + ctx.simplify_ref_deref(rvalue); + ctx.simplify_ptr_aggregate(rvalue); ctx.simplify_cast(rvalue); } _ => {} @@ -66,27 +64,12 @@ impl<'tcx> crate::MirPass<'tcx> for InstSimplify { struct InstSimplifyContext<'a, 'tcx> { tcx: TyCtxt<'tcx>, local_decls: &'a LocalDecls<'tcx>, - param_env: ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, } impl<'tcx> InstSimplifyContext<'_, 'tcx> { - fn should_simplify(&self, source_info: &SourceInfo, rvalue: &Rvalue<'tcx>) -> bool { - self.should_simplify_custom(source_info, "Rvalue", rvalue) - } - - fn should_simplify_custom( - &self, - source_info: &SourceInfo, - label: &str, - value: impl std::fmt::Debug, - ) -> bool { - self.tcx.consider_optimizing(|| { - format!("InstSimplify - {label}: {value:?} SourceInfo: {source_info:?}") - }) - } - /// Transform boolean comparisons into logical operations. - fn simplify_bool_cmp(&self, source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) { + fn simplify_bool_cmp(&self, rvalue: &mut Rvalue<'tcx>) { match rvalue { Rvalue::BinaryOp(op @ (BinOp::Eq | BinOp::Ne), box (a, b)) => { let new = match (op, self.try_eval_bool(a), self.try_eval_bool(b)) { @@ -117,9 +100,7 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> { _ => None, }; - if let Some(new) = new - && self.should_simplify(source_info, rvalue) - { + if let Some(new) = new { *rvalue = new; } } @@ -134,17 +115,13 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> { } /// Transform `&(*a)` ==> `a`. - fn simplify_ref_deref(&self, source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) { + fn simplify_ref_deref(&self, rvalue: &mut Rvalue<'tcx>) { if let Rvalue::Ref(_, _, place) | Rvalue::RawPtr(_, place) = rvalue { if let Some((base, ProjectionElem::Deref)) = place.as_ref().last_projection() { if rvalue.ty(self.local_decls, self.tcx) != base.ty(self.local_decls, self.tcx).ty { return; } - if !self.should_simplify(source_info, rvalue) { - return; - } - *rvalue = Rvalue::Use(Operand::Copy(Place { local: base.local, projection: self.tcx.mk_place_elems(base.projection), @@ -153,37 +130,13 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> { } } - /// Transform `Len([_; N])` ==> `N`. - fn simplify_len(&self, source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) { - if let Rvalue::Len(ref place) = *rvalue { - let place_ty = place.ty(self.local_decls, self.tcx).ty; - if let ty::Array(_, len) = *place_ty.kind() { - if !self.should_simplify(source_info, rvalue) { - return; - } - - let const_ = Const::from_ty_const(len, self.tcx.types.usize, self.tcx); - let constant = ConstOperand { span: source_info.span, const_, user_ty: None }; - *rvalue = Rvalue::Use(Operand::Constant(Box::new(constant))); - } - } - } - /// Transform `Aggregate(RawPtr, [p, ()])` ==> `Cast(PtrToPtr, p)`. - fn simplify_ptr_aggregate(&self, source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) { + fn simplify_ptr_aggregate(&self, rvalue: &mut Rvalue<'tcx>) { if let Rvalue::Aggregate(box AggregateKind::RawPtr(pointee_ty, mutability), fields) = rvalue { let meta_ty = fields.raw[1].ty(self.local_decls, self.tcx); if meta_ty.is_unit() { // The mutable borrows we're holding prevent printing `rvalue` here - if !self.should_simplify_custom( - source_info, - "Aggregate::RawPtr", - (&pointee_ty, *mutability, &fields), - ) { - return; - } - let mut fields = std::mem::take(fields); let _meta = fields.pop().unwrap(); let data = fields.pop().unwrap(); @@ -193,10 +146,10 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> { } } - fn simplify_ub_check(&self, source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) { + fn simplify_ub_check(&self, rvalue: &mut Rvalue<'tcx>) { if let Rvalue::NullaryOp(NullOp::UbChecks, _) = *rvalue { let const_ = Const::from_bool(self.tcx, self.tcx.sess.ub_checks()); - let constant = ConstOperand { span: source_info.span, const_, user_ty: None }; + let constant = ConstOperand { span: DUMMY_SP, const_, user_ty: None }; *rvalue = Rvalue::Use(Operand::Constant(Box::new(constant))); } } @@ -284,16 +237,6 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> { return; } - if !self.tcx.consider_optimizing(|| { - format!( - "InstSimplify - Call: {:?} SourceInfo: {:?}", - (fn_def_id, fn_args), - terminator.source_info - ) - }) { - return; - } - let Ok([arg]) = take_array(args) else { return }; let Some(arg_place) = arg.node.place() else { return }; @@ -348,7 +291,7 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> { } let known_is_valid = - intrinsic_assert_panics(self.tcx, self.param_env, args[0], intrinsic_name); + intrinsic_assert_panics(self.tcx, self.typing_env, args[0], intrinsic_name); match known_is_valid { // We don't know the layout or it's not validity assertion at all, don't touch it None => {} @@ -366,13 +309,13 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> { fn intrinsic_assert_panics<'tcx>( tcx: TyCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, arg: ty::GenericArg<'tcx>, intrinsic_name: Symbol, ) -> Option<bool> { let requirement = ValidityRequirement::from_intrinsic(intrinsic_name)?; let ty = arg.expect_ty(); - Some(!tcx.check_validity_requirement((requirement, param_env.and(ty))).ok()?) + Some(!tcx.check_validity_requirement((requirement, typing_env.as_query_input(ty))).ok()?) } fn resolve_rust_intrinsic<'tcx>( diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index 3772589ac4e..8feb90ff7a0 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -35,7 +35,6 @@ //! Likewise, applying the optimisation can create a lot of new MIR, so we bound the instruction //! cost by `MAX_COST`. -use rustc_abi::{TagEncoding, Variants}; use rustc_arena::DroplessArena; use rustc_const_eval::const_eval::DummyMachine; use rustc_const_eval::interpret::{ImmTy, Immediate, InterpCx, OpTy, Projectable}; @@ -77,13 +76,12 @@ impl<'tcx> crate::MirPass<'tcx> for JumpThreading { return; } - let param_env = tcx.param_env_reveal_all_normalized(def_id); - + let typing_env = body.typing_env(tcx); let arena = &DroplessArena::default(); let mut finder = TOFinder { tcx, - param_env, - ecx: InterpCx::new(tcx, DUMMY_SP, param_env, DummyMachine), + typing_env, + ecx: InterpCx::new(tcx, DUMMY_SP, typing_env, DummyMachine), body, arena, map: Map::new(tcx, body, Some(MAX_PLACES)), @@ -119,7 +117,7 @@ struct ThreadingOpportunity { struct TOFinder<'a, 'tcx> { tcx: TyCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, ecx: InterpCx<'tcx, DummyMachine>, body: &'a Body<'tcx>, map: Map<'tcx>, @@ -207,7 +205,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { let Some(discr) = self.map.find(discr.as_ref()) else { return }; debug!(?discr); - let cost = CostChecker::new(self.tcx, self.param_env, None, self.body); + let cost = CostChecker::new(self.tcx, self.typing_env, None, self.body); let mut state = State::new_reachable(); let conds = if let Some((value, then, else_)) = targets.as_static_if() { @@ -353,6 +351,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { | StatementKind::FakeRead(..) | StatementKind::ConstEvalCounter | StatementKind::PlaceMention(..) + | StatementKind::BackwardIncompatibleDropHint { .. } | StatementKind::Nop => None, } } @@ -528,7 +527,8 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { // Avoid handling them, though this could be extended in the future. return; } - let Some(value) = value.const_.try_eval_scalar_int(self.tcx, self.param_env) else { + let Some(value) = value.const_.try_eval_scalar_int(self.tcx, self.typing_env) + else { return; }; let conds = conditions.map(self.arena, |c| Condition { @@ -564,31 +564,15 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { StatementKind::SetDiscriminant { box place, variant_index } => { let Some(discr_target) = self.map.find_discr(place.as_ref()) else { return }; let enum_ty = place.ty(self.body, self.tcx).ty; - // `SetDiscriminant` may be a no-op if the assigned variant is the untagged variant - // of a niche encoding. If we cannot ensure that we write to the discriminant, do - // nothing. - let Ok(enum_layout) = self.ecx.layout_of(enum_ty) else { + // `SetDiscriminant` guarantees that the discriminant is now `variant_index`. + // Even if the discriminant write does nothing due to niches, it is UB to set the + // discriminant when the data does not encode the desired discriminant. + let Some(discr) = + self.ecx.discriminant_for_variant(enum_ty, *variant_index).discard_err() + else { return; }; - let writes_discriminant = match enum_layout.variants { - Variants::Single { index } => { - assert_eq!(index, *variant_index); - true - } - Variants::Multiple { tag_encoding: TagEncoding::Direct, .. } => true, - Variants::Multiple { - tag_encoding: TagEncoding::Niche { untagged_variant, .. }, - .. - } => *variant_index != untagged_variant, - }; - if writes_discriminant { - let Some(discr) = - self.ecx.discriminant_for_variant(enum_ty, *variant_index).discard_err() - else { - return; - }; - self.process_immediate(bb, discr_target, discr, state); - } + self.process_immediate(bb, discr_target, discr, state); } // If we expect `lhs ?= true`, we have an opportunity if we assume `lhs == true`. StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume( diff --git a/compiler/rustc_mir_transform/src/known_panics_lint.rs b/compiler/rustc_mir_transform/src/known_panics_lint.rs index 0604665642a..acf3eb2b62c 100644 --- a/compiler/rustc_mir_transform/src/known_panics_lint.rs +++ b/compiler/rustc_mir_transform/src/known_panics_lint.rs @@ -18,7 +18,7 @@ use rustc_middle::bug; use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::layout::{LayoutError, LayoutOf, LayoutOfHelpers, TyAndLayout}; -use rustc_middle::ty::{self, ConstInt, ParamEnv, ScalarInt, Ty, TyCtxt, TypeVisitableExt}; +use rustc_middle::ty::{self, ConstInt, ScalarInt, Ty, TyCtxt, TypeVisitableExt}; use rustc_span::Span; use tracing::{debug, instrument, trace}; @@ -65,7 +65,7 @@ impl<'tcx> crate::MirLint<'tcx> for KnownPanicsLint { struct ConstPropagator<'mir, 'tcx> { ecx: InterpCx<'tcx, DummyMachine>, tcx: TyCtxt<'tcx>, - param_env: ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, worklist: Vec<BasicBlock>, visited_blocks: BitSet<BasicBlock>, locals: IndexVec<Local, Value<'tcx>>, @@ -169,25 +169,26 @@ impl<'tcx> ty::layout::HasTyCtxt<'tcx> for ConstPropagator<'_, 'tcx> { } } -impl<'tcx> ty::layout::HasParamEnv<'tcx> for ConstPropagator<'_, 'tcx> { +impl<'tcx> ty::layout::HasTypingEnv<'tcx> for ConstPropagator<'_, 'tcx> { #[inline] - fn param_env(&self) -> ty::ParamEnv<'tcx> { - self.param_env + fn typing_env(&self) -> ty::TypingEnv<'tcx> { + self.typing_env } } impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { fn new(body: &'mir Body<'tcx>, tcx: TyCtxt<'tcx>) -> ConstPropagator<'mir, 'tcx> { let def_id = body.source.def_id(); - let param_env = tcx.param_env_reveal_all_normalized(def_id); - - let can_const_prop = CanConstProp::check(tcx, param_env, body); - let ecx = InterpCx::new(tcx, tcx.def_span(def_id), param_env, DummyMachine); + // FIXME(#132279): This is used during the phase transition from analysis + // to runtime, so we have to manually specify the correct typing mode. + let typing_env = ty::TypingEnv::post_analysis(tcx, body.source.def_id()); + let can_const_prop = CanConstProp::check(tcx, typing_env, body); + let ecx = InterpCx::new(tcx, tcx.def_span(def_id), typing_env, DummyMachine); ConstPropagator { ecx, tcx, - param_env, + typing_env, worklist: vec![START_BLOCK], visited_blocks: BitSet::new_empty(body.basic_blocks.len()), locals: IndexVec::from_elem_n(Value::Uninit, body.local_decls.len()), @@ -257,10 +258,10 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { // Normalization needed b/c known panics lint runs in // `mir_drops_elaborated_and_const_checked`, which happens before // optimized MIR. Only after optimizing the MIR can we guarantee - // that the `RevealAll` pass has happened and that the body's consts + // that the `PostAnalysisNormalize` pass has happened and that the body's consts // are normalized, so any call to resolve before that needs to be // manually normalized. - let val = self.tcx.try_normalize_erasing_regions(self.param_env, c.const_).ok()?; + let val = self.tcx.try_normalize_erasing_regions(self.typing_env, c.const_).ok()?; self.use_ecx(|this| this.ecx.eval_mir_constant(&val, c.span, None))? .as_mplace_or_imm() @@ -450,7 +451,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { if rvalue.has_param() { return None; } - if !rvalue.ty(self.local_decls(), self.tcx).is_sized(self.tcx, self.param_env) { + if !rvalue.ty(self.local_decls(), self.tcx).is_sized(self.tcx, self.typing_env) { // the interpreter doesn't support unsized locals (only unsized arguments), // but rustc does (in a kinda broken way), so we have to skip them here return None; @@ -622,7 +623,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { NullOp::AlignOf => op_layout.align.abi.bytes(), NullOp::OffsetOf(fields) => self .tcx - .offset_of_subfield(self.param_env, op_layout, fields.iter()) + .offset_of_subfield(self.typing_env, op_layout, fields.iter()) .bytes(), NullOp::UbChecks => return None, }; @@ -873,7 +874,7 @@ impl CanConstProp { /// Returns true if `local` can be propagated fn check<'tcx>( tcx: TyCtxt<'tcx>, - param_env: ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, body: &Body<'tcx>, ) -> IndexVec<Local, ConstPropMode> { let mut cpv = CanConstProp { @@ -888,7 +889,7 @@ impl CanConstProp { // variant of a union *val = ConstPropMode::NoPropagation; } else { - match tcx.layout_of(param_env.and(ty)) { + match tcx.layout_of(typing_env.as_query_input(ty)) { Ok(layout) if layout.size < Size::from_bytes(MAX_ALLOC_LIMIT) => {} // Either the layout fails to compute, then we can't use this local anyway // or the local is too large, then we don't want to. diff --git a/compiler/rustc_mir_transform/src/large_enums.rs b/compiler/rustc_mir_transform/src/large_enums.rs index fa659a56a27..490e7dd8f7e 100644 --- a/compiler/rustc_mir_transform/src/large_enums.rs +++ b/compiler/rustc_mir_transform/src/large_enums.rs @@ -3,7 +3,7 @@ use rustc_data_structures::fx::FxHashMap; use rustc_middle::mir::interpret::AllocId; use rustc_middle::mir::*; use rustc_middle::ty::util::IntTypeExt; -use rustc_middle::ty::{self, AdtDef, ParamEnv, Ty, TyCtxt}; +use rustc_middle::ty::{self, AdtDef, Ty, TyCtxt}; use rustc_session::Session; /// A pass that seeks to optimize unnecessary moves of large enum types, if there is a large @@ -39,8 +39,7 @@ impl<'tcx> crate::MirPass<'tcx> for EnumSizeOpt { // platform, but it will still be valid. let mut alloc_cache = FxHashMap::default(); - let body_did = body.source.def_id(); - let param_env = tcx.param_env_reveal_all_normalized(body_did); + let typing_env = body.typing_env(tcx); let blocks = body.basic_blocks.as_mut(); let local_decls = &mut body.local_decls; @@ -58,7 +57,7 @@ impl<'tcx> crate::MirPass<'tcx> for EnumSizeOpt { let ty = lhs.ty(local_decls, tcx).ty; let (adt_def, num_variants, alloc_id) = - self.candidate(tcx, param_env, ty, &mut alloc_cache)?; + self.candidate(tcx, typing_env, ty, &mut alloc_cache)?; let source_info = st.source_info; let span = source_info.span; @@ -207,7 +206,7 @@ impl EnumSizeOpt { fn candidate<'tcx>( &self, tcx: TyCtxt<'tcx>, - param_env: ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, ty: Ty<'tcx>, alloc_cache: &mut FxHashMap<Ty<'tcx>, AllocId>, ) -> Option<(AdtDef<'tcx>, usize, AllocId)> { @@ -215,9 +214,9 @@ impl EnumSizeOpt { ty::Adt(adt_def, _args) if adt_def.is_enum() => adt_def, _ => return None, }; - let layout = tcx.layout_of(param_env.and(ty)).ok()?; + let layout = tcx.layout_of(typing_env.as_query_input(ty)).ok()?; let variants = match &layout.variants { - Variants::Single { .. } => return None, + Variants::Single { .. } | Variants::Empty => return None, Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, .. } => return None, Variants::Multiple { variants, .. } if variants.len() <= 1 => return None, diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index d184328748f..5c090bf7cad 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -9,7 +9,6 @@ #![feature(let_chains)] #![feature(map_try_insert)] #![feature(never_type)] -#![feature(round_char_boundary)] #![feature(try_blocks)] #![feature(yeet_expr)] #![warn(unreachable_pub)] @@ -41,77 +40,160 @@ use tracing::{debug, trace}; #[macro_use] mod pass_manager; +use std::sync::LazyLock; + use pass_manager::{self as pm, Lint, MirLint, MirPass, WithMinOptLevel}; -mod abort_unwinding_calls; -mod add_call_guards; -mod add_moves_for_packed_drops; -mod add_retag; -mod add_subtyping_projections; -mod check_alignment; -mod check_const_item_mutation; -mod check_packed_ref; -mod check_undefined_transmutes; -// This pass is public to allow external drivers to perform MIR cleanup -pub mod cleanup_post_borrowck; -mod copy_prop; -mod coroutine; mod cost_checker; -mod coverage; mod cross_crate_inline; -mod ctfe_limit; -mod dataflow_const_prop; -mod dead_store_elimination; mod deduce_param_attrs; -mod deduplicate_blocks; -mod deref_separator; -mod dest_prop; -pub mod dump_mir; -mod early_otherwise_branch; -mod elaborate_box_derefs; -mod elaborate_drops; mod errors; mod ffi_unwind_calls; -mod function_item_references; -mod gvn; -// Made public so that `mir_drops_elaborated_and_const_checked` can be overridden -// by custom rustc drivers, running all the steps by themselves. See #114628. -pub mod inline; -mod instsimplify; -mod jump_threading; -mod known_panics_lint; -mod large_enums; mod lint; -mod lower_intrinsics; -mod lower_slice_len; -mod match_branches; -mod mentioned_items; -mod multiple_return_terminators; -mod nrvo; -mod post_drop_elaboration; -mod prettify; -mod promote_consts; -mod ref_prop; -mod remove_noop_landing_pads; -mod remove_place_mention; -mod remove_storage_markers; -mod remove_uninit_drops; -mod remove_unneeded_drops; -mod remove_zsts; -mod required_consts; -mod reveal_all; -mod sanity_check; +mod lint_tail_expr_drop_order; mod shim; mod ssa; -// This pass is public to allow external drivers to perform MIR cleanup -pub mod simplify; -mod simplify_branches; -mod simplify_comparison_integral; -mod single_use_consts; -mod sroa; -mod unreachable_enum_branching; -mod unreachable_prop; -mod validate; + +/// We import passes via this macro so that we can have a static list of pass names +/// (used to verify CLI arguments). It takes a list of modules, followed by the passes +/// declared within them. +/// ```ignore,macro-test +/// declare_passes! { +/// // Declare a single pass from the module `abort_unwinding_calls` +/// mod abort_unwinding_calls : AbortUnwindingCalls; +/// // When passes are grouped together as an enum, declare the two constituent passes +/// mod add_call_guards : AddCallGuards { +/// AllCallEdges, +/// CriticalCallEdges +/// }; +/// // Declares multiple pass groups, each containing their own constituent passes +/// mod simplify : SimplifyCfg { +/// Initial, +/// /* omitted */ +/// }, SimplifyLocals { +/// BeforeConstProp, +/// /* omitted */ +/// }; +/// } +/// ``` +macro_rules! declare_passes { + ( + $( + $vis:vis mod $mod_name:ident : $($pass_name:ident $( { $($ident:ident),* } )?),+ $(,)?; + )* + ) => { + $( + $vis mod $mod_name; + $( + // Make sure the type name is correct + #[allow(unused_imports)] + use $mod_name::$pass_name as _; + )+ + )* + + static PASS_NAMES: LazyLock<FxIndexSet<&str>> = LazyLock::new(|| [ + // Fake marker pass + "PreCodegen", + $( + $( + stringify!($pass_name), + $( + $( + $mod_name::$pass_name::$ident.name(), + )* + )? + )+ + )* + ].into_iter().collect()); + }; +} + +declare_passes! { + mod abort_unwinding_calls : AbortUnwindingCalls; + mod add_call_guards : AddCallGuards { AllCallEdges, CriticalCallEdges }; + mod add_moves_for_packed_drops : AddMovesForPackedDrops; + mod add_retag : AddRetag; + mod add_subtyping_projections : Subtyper; + mod check_alignment : CheckAlignment; + mod check_const_item_mutation : CheckConstItemMutation; + mod check_packed_ref : CheckPackedRef; + mod check_undefined_transmutes : CheckUndefinedTransmutes; + // This pass is public to allow external drivers to perform MIR cleanup + pub mod cleanup_post_borrowck : CleanupPostBorrowck; + + mod copy_prop : CopyProp; + mod coroutine : StateTransform; + mod coverage : InstrumentCoverage; + mod ctfe_limit : CtfeLimit; + mod dataflow_const_prop : DataflowConstProp; + mod dead_store_elimination : DeadStoreElimination { + Initial, + Final + }; + mod deduplicate_blocks : DeduplicateBlocks; + mod deref_separator : Derefer; + mod dest_prop : DestinationPropagation; + pub mod dump_mir : Marker; + mod early_otherwise_branch : EarlyOtherwiseBranch; + mod elaborate_box_derefs : ElaborateBoxDerefs; + mod elaborate_drops : ElaborateDrops; + mod function_item_references : FunctionItemReferences; + mod gvn : GVN; + // Made public so that `mir_drops_elaborated_and_const_checked` can be overridden + // by custom rustc drivers, running all the steps by themselves. See #114628. + pub mod inline : Inline; + mod instsimplify : InstSimplify { BeforeInline, AfterSimplifyCfg }; + mod jump_threading : JumpThreading; + mod known_panics_lint : KnownPanicsLint; + mod large_enums : EnumSizeOpt; + mod lower_intrinsics : LowerIntrinsics; + mod lower_slice_len : LowerSliceLenCalls; + mod match_branches : MatchBranchSimplification; + mod mentioned_items : MentionedItems; + mod multiple_return_terminators : MultipleReturnTerminators; + mod nrvo : RenameReturnPlace; + mod post_drop_elaboration : CheckLiveDrops; + mod prettify : ReorderBasicBlocks, ReorderLocals; + mod promote_consts : PromoteTemps; + mod ref_prop : ReferencePropagation; + mod remove_noop_landing_pads : RemoveNoopLandingPads; + mod remove_place_mention : RemovePlaceMention; + mod remove_storage_markers : RemoveStorageMarkers; + mod remove_uninit_drops : RemoveUninitDrops; + mod remove_unneeded_drops : RemoveUnneededDrops; + mod remove_zsts : RemoveZsts; + mod required_consts : RequiredConstsVisitor; + mod post_analysis_normalize : PostAnalysisNormalize; + mod sanity_check : SanityCheck; + // This pass is public to allow external drivers to perform MIR cleanup + pub mod simplify : + SimplifyCfg { + Initial, + PromoteConsts, + RemoveFalseEdges, + PostAnalysis, + PreOptimizations, + Final, + MakeShim, + AfterUnreachableEnumBranching + }, + SimplifyLocals { + BeforeConstProp, + AfterGVN, + Final + }; + mod simplify_branches : SimplifyConstCondition { + AfterConstProp, + Final + }; + mod simplify_comparison_integral : SimplifyComparisonIntegral; + mod single_use_consts : SingleUseConsts; + mod sroa : ScalarReplacementOfAggregates; + mod strip_debuginfo : StripDebugInfo; + mod unreachable_enum_branching : UnreachableEnumBranching; + mod unreachable_prop : UnreachablePropagation; + mod validate : Validator; +} rustc_fluent_macro::fluent_messages! { "../messages.ftl" } @@ -253,10 +335,14 @@ fn mir_keys(tcx: TyCtxt<'_>, (): ()) -> FxIndexSet<LocalDefId> { } fn mir_const_qualif(tcx: TyCtxt<'_>, def: LocalDefId) -> ConstQualifs { - let const_kind = tcx.hir().body_const_context(def); - + // N.B., this `borrow()` is guaranteed to be valid (i.e., the value + // cannot yet be stolen), because `mir_promoted()`, which steals + // from `mir_built()`, forces this query to execute before + // performing the steal. + let body = &tcx.mir_built(def).borrow(); + let ccx = check_consts::ConstCx::new(tcx, body); // No need to const-check a non-const `fn`. - match const_kind { + match ccx.const_kind { Some(ConstContext::Const { .. } | ConstContext::Static(_) | ConstContext::ConstFn) => {} None => span_bug!( tcx.def_span(def), @@ -264,20 +350,12 @@ fn mir_const_qualif(tcx: TyCtxt<'_>, def: LocalDefId) -> ConstQualifs { ), } - // N.B., this `borrow()` is guaranteed to be valid (i.e., the value - // cannot yet be stolen), because `mir_promoted()`, which steals - // from `mir_built()`, forces this query to execute before - // performing the steal. - let body = &tcx.mir_built(def).borrow(); - if body.return_ty().references_error() { // It's possible to reach here without an error being emitted (#121103). tcx.dcx().span_delayed_bug(body.span, "mir_const_qualif: MIR had errors"); return Default::default(); } - let ccx = check_consts::ConstCx { body, tcx, const_kind, param_env: tcx.param_env(def) }; - let mut validator = check_consts::check::Checker::new(&ccx); validator.check_body(); @@ -414,6 +492,7 @@ fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> & } let (body, _) = tcx.mir_promoted(def); + lint_tail_expr_drop_order::run_lint(tcx, def, &body.borrow()); let mut body = body.steal(); if let Some(error_reported) = tainted_by_errors { @@ -526,8 +605,8 @@ fn run_runtime_lowering_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // These next passes must be executed together. &add_call_guards::CriticalCallEdges, // Must be done before drop elaboration because we need to drop opaque types, too. - &reveal_all::RevealAll, - // Calling this after reveal_all ensures that we don't deal with opaque types. + &post_analysis_normalize::PostAnalysisNormalize, + // Calling this after `PostAnalysisNormalize` ensures that we don't deal with opaque types. &add_subtyping_projections::Subtyper, &elaborate_drops::ElaborateDrops, // This will remove extraneous landing pads which are no longer @@ -621,6 +700,8 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { &o1(simplify_branches::SimplifyConstCondition::Final), &o1(remove_noop_landing_pads::RemoveNoopLandingPads), &o1(simplify::SimplifyCfg::Final), + // After the last SimplifyCfg, because this wants one-block functions. + &strip_debuginfo::StripDebugInfo, ©_prop::CopyProp, &dead_store_elimination::DeadStoreElimination::Final, &nrvo::RenameReturnPlace, diff --git a/compiler/rustc_mir_transform/src/lint.rs b/compiler/rustc_mir_transform/src/lint.rs index d8ff1cfc90b..29e762af8de 100644 --- a/compiler/rustc_mir_transform/src/lint.rs +++ b/compiler/rustc_mir_transform/src/lint.rs @@ -9,8 +9,7 @@ use rustc_index::bit_set::BitSet; use rustc_middle::mir::visit::{PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; -use rustc_mir_dataflow::impls::{MaybeStorageDead, MaybeStorageLive}; -use rustc_mir_dataflow::storage::always_storage_live_locals; +use rustc_mir_dataflow::impls::{MaybeStorageDead, MaybeStorageLive, always_storage_live_locals}; use rustc_mir_dataflow::{Analysis, ResultsCursor}; pub(super) fn lint_body<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, when: String) { diff --git a/compiler/rustc_mir_transform/src/lint_tail_expr_drop_order.rs b/compiler/rustc_mir_transform/src/lint_tail_expr_drop_order.rs new file mode 100644 index 00000000000..7fb421dea0c --- /dev/null +++ b/compiler/rustc_mir_transform/src/lint_tail_expr_drop_order.rs @@ -0,0 +1,699 @@ +use std::cell::RefCell; +use std::collections::hash_map; +use std::rc::Rc; + +use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use rustc_data_structures::unord::{UnordMap, UnordSet}; +use rustc_errors::Subdiagnostic; +use rustc_hir::CRATE_HIR_ID; +use rustc_hir::def_id::{DefId, LocalDefId}; +use rustc_index::bit_set::MixedBitSet; +use rustc_index::{IndexSlice, IndexVec}; +use rustc_macros::{LintDiagnostic, Subdiagnostic}; +use rustc_middle::bug; +use rustc_middle::mir::{ + self, BasicBlock, Body, ClearCrossCrate, Local, Location, Place, StatementKind, TerminatorKind, + dump_mir, +}; +use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_mir_dataflow::impls::MaybeInitializedPlaces; +use rustc_mir_dataflow::move_paths::{LookupResult, MoveData, MovePathIndex}; +use rustc_mir_dataflow::{Analysis, MaybeReachable, ResultsCursor}; +use rustc_session::lint::builtin::TAIL_EXPR_DROP_ORDER; +use rustc_session::lint::{self}; +use rustc_span::{DUMMY_SP, Span, Symbol}; +use rustc_type_ir::data_structures::IndexMap; +use smallvec::{SmallVec, smallvec}; +use tracing::{debug, instrument}; + +fn place_has_common_prefix<'tcx>(left: &Place<'tcx>, right: &Place<'tcx>) -> bool { + left.local == right.local + && left.projection.iter().zip(right.projection).all(|(left, right)| left == right) +} + +/// Cache entry of `drop` at a `BasicBlock` +#[derive(Debug, Clone, Copy)] +enum MovePathIndexAtBlock { + /// We know nothing yet + Unknown, + /// We know that the `drop` here has no effect + None, + /// We know that the `drop` here will invoke a destructor + Some(MovePathIndex), +} + +struct DropsReachable<'a, 'mir, 'tcx> { + body: &'a Body<'tcx>, + place: &'a Place<'tcx>, + drop_span: &'a mut Option<Span>, + move_data: &'a MoveData<'tcx>, + maybe_init: &'a mut ResultsCursor<'mir, 'tcx, MaybeInitializedPlaces<'mir, 'tcx>>, + block_drop_value_info: &'a mut IndexSlice<BasicBlock, MovePathIndexAtBlock>, + collected_drops: &'a mut MixedBitSet<MovePathIndex>, + visited: FxHashMap<BasicBlock, Rc<RefCell<MixedBitSet<MovePathIndex>>>>, +} + +impl<'a, 'mir, 'tcx> DropsReachable<'a, 'mir, 'tcx> { + fn visit(&mut self, block: BasicBlock) { + let move_set_size = self.move_data.move_paths.len(); + let make_new_path_set = || Rc::new(RefCell::new(MixedBitSet::new_empty(move_set_size))); + + let data = &self.body.basic_blocks[block]; + let Some(terminator) = &data.terminator else { return }; + // Given that we observe these dropped locals here at `block` so far, we will try to update + // the successor blocks. An occupied entry at `block` in `self.visited` signals that we + // have visited `block` before. + let dropped_local_here = + Rc::clone(self.visited.entry(block).or_insert_with(make_new_path_set)); + // We could have invoked reverse lookup for a `MovePathIndex` every time, but unfortunately + // it is expensive. Let's cache them in `self.block_drop_value_info`. + match self.block_drop_value_info[block] { + MovePathIndexAtBlock::Some(dropped) => { + dropped_local_here.borrow_mut().insert(dropped); + } + MovePathIndexAtBlock::Unknown => { + if let TerminatorKind::Drop { place, .. } = &terminator.kind + && let LookupResult::Exact(idx) | LookupResult::Parent(Some(idx)) = + self.move_data.rev_lookup.find(place.as_ref()) + { + // Since we are working with MIRs at a very early stage, observing a `drop` + // terminator is not indicative enough that the drop will definitely happen. + // That is decided in the drop elaboration pass instead. Therefore, we need to + // consult with the maybe-initialization information. + self.maybe_init.seek_before_primary_effect(Location { + block, + statement_index: data.statements.len(), + }); + + // Check if the drop of `place` under inspection is really in effect. This is + // true only when `place` may have been initialized along a control flow path + // from a BID to the drop program point today. In other words, this is where + // the drop of `place` will happen in the future instead. + if let MaybeReachable::Reachable(maybe_init) = self.maybe_init.get() + && maybe_init.contains(idx) + { + // We also cache the drop information, so that we do not need to check on + // data-flow cursor again. + self.block_drop_value_info[block] = MovePathIndexAtBlock::Some(idx); + dropped_local_here.borrow_mut().insert(idx); + } else { + self.block_drop_value_info[block] = MovePathIndexAtBlock::None; + } + } + } + MovePathIndexAtBlock::None => {} + } + + for succ in terminator.successors() { + let target = &self.body.basic_blocks[succ]; + if target.is_cleanup { + continue; + } + + // As long as we are passing through a new block, or new dropped places to propagate, + // we will proceed with `succ` + let dropped_local_there = match self.visited.entry(succ) { + hash_map::Entry::Occupied(occupied_entry) => { + if succ == block + || !occupied_entry.get().borrow_mut().union(&*dropped_local_here.borrow()) + { + // `succ` has been visited but no new drops observed so far, + // so we can bail on `succ` until new drop information arrives + continue; + } + Rc::clone(occupied_entry.get()) + } + hash_map::Entry::Vacant(vacant_entry) => Rc::clone( + vacant_entry.insert(Rc::new(RefCell::new(dropped_local_here.borrow().clone()))), + ), + }; + if let Some(terminator) = &target.terminator + && let TerminatorKind::Drop { + place: dropped_place, + target: _, + unwind: _, + replace: _, + } = &terminator.kind + && place_has_common_prefix(dropped_place, self.place) + { + // We have now reached the current drop of the `place`. + // Let's check the observed dropped places in. + self.collected_drops.union(&*dropped_local_there.borrow()); + if self.drop_span.is_none() { + // FIXME(@dingxiangfei2009): it turns out that `self.body.source_scopes` are + // still a bit wonky. There is a high chance that this span still points to a + // block rather than a statement semicolon. + *self.drop_span = Some(terminator.source_info.span); + } + // Now we have discovered a simple control flow path from a future drop point + // to the current drop point. + // We will not continue from there. + } else { + self.visit(succ) + } + } + } +} + +/// An additional filter to exclude well-known types from the ecosystem +/// because their drops are trivial. +/// This returns additional types to check if the drops are delegated to those. +/// A typical example is `hashbrown::HashMap<K, V>`, whose drop is delegated to `K` and `V`. +fn true_significant_drop_ty<'tcx>( + tcx: TyCtxt<'tcx>, + ty: Ty<'tcx>, +) -> Option<SmallVec<[Ty<'tcx>; 2]>> { + if let ty::Adt(def, args) = ty.kind() { + let mut did = def.did(); + let mut name_rev = vec![]; + loop { + let key = tcx.def_key(did); + + match key.disambiguated_data.data { + rustc_hir::definitions::DefPathData::CrateRoot => { + name_rev.push(tcx.crate_name(did.krate)) + } + rustc_hir::definitions::DefPathData::TypeNs(symbol) => name_rev.push(symbol), + _ => return None, + } + if let Some(parent) = key.parent { + did = DefId { krate: did.krate, index: parent }; + } else { + break; + } + } + let name_str: Vec<_> = name_rev.iter().rev().map(|x| x.as_str()).collect(); + debug!(?name_str); + match name_str[..] { + // These are the types from Rust core ecosystem + ["sym" | "proc_macro2", ..] + | ["core" | "std", "task", "LocalWaker" | "Waker"] + | ["core" | "std", "task", "wake", "LocalWaker" | "Waker"] => Some(smallvec![]), + // These are important types from Rust ecosystem + ["tracing", "instrument", "Instrumented"] | ["bytes", "Bytes"] => Some(smallvec![]), + ["hashbrown", "raw", "RawTable" | "RawIntoIter"] => { + if let [ty, ..] = &***args + && let Some(ty) = ty.as_type() + { + Some(smallvec![ty]) + } else { + None + } + } + ["hashbrown", "raw", "RawDrain"] => { + if let [_, ty, ..] = &***args + && let Some(ty) = ty.as_type() + { + Some(smallvec![ty]) + } else { + None + } + } + _ => None, + } + } else { + None + } +} + +/// Returns the list of types with a "potentially sigificant" that may be dropped +/// by dropping a value of type `ty`. +#[instrument(level = "debug", skip(tcx, typing_env))] +fn extract_component_raw<'tcx>( + tcx: TyCtxt<'tcx>, + typing_env: ty::TypingEnv<'tcx>, + ty: Ty<'tcx>, + ty_seen: &mut UnordSet<Ty<'tcx>>, +) -> SmallVec<[Ty<'tcx>; 4]> { + // Droppiness does not depend on regions, so let us erase them. + let ty = tcx.try_normalize_erasing_regions(typing_env, ty).unwrap_or(ty); + + let tys = tcx.list_significant_drop_tys(typing_env.as_query_input(ty)); + debug!(?ty, "components"); + let mut out_tys = smallvec![]; + for ty in tys { + if let Some(tys) = true_significant_drop_ty(tcx, ty) { + // Some types can be further opened up because the drop is simply delegated + for ty in tys { + if ty_seen.insert(ty) { + out_tys.extend(extract_component_raw(tcx, typing_env, ty, ty_seen)); + } + } + } else { + if ty_seen.insert(ty) { + out_tys.push(ty); + } + } + } + out_tys +} + +#[instrument(level = "debug", skip(tcx, typing_env))] +fn extract_component_with_significant_dtor<'tcx>( + tcx: TyCtxt<'tcx>, + typing_env: ty::TypingEnv<'tcx>, + ty: Ty<'tcx>, +) -> SmallVec<[Ty<'tcx>; 4]> { + let mut tys = extract_component_raw(tcx, typing_env, ty, &mut Default::default()); + let mut deduplicate = FxHashSet::default(); + tys.retain(|oty| deduplicate.insert(*oty)); + tys.into_iter().collect() +} + +/// Extract the span of the custom destructor of a type +/// especially the span of the `impl Drop` header or its entire block +/// when we are working with current local crate. +#[instrument(level = "debug", skip(tcx))] +fn ty_dtor_span<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option<Span> { + match ty.kind() { + ty::Bool + | ty::Char + | ty::Int(_) + | ty::Uint(_) + | ty::Float(_) + | ty::Error(_) + | ty::Str + | ty::Never + | ty::RawPtr(_, _) + | ty::Ref(_, _, _) + | ty::FnPtr(_, _) + | ty::Tuple(_) + | ty::Dynamic(_, _, _) + | ty::Alias(_, _) + | ty::Bound(_, _) + | ty::Pat(_, _) + | ty::Placeholder(_) + | ty::Infer(_) + | ty::Slice(_) + | ty::Array(_, _) => None, + ty::Adt(adt_def, _) => { + let did = adt_def.did(); + let try_local_did_span = |did: DefId| { + if let Some(local) = did.as_local() { + tcx.source_span(local) + } else { + tcx.def_span(did) + } + }; + let dtor = if let Some(dtor) = tcx.adt_destructor(did) { + dtor.did + } else if let Some(dtor) = tcx.adt_async_destructor(did) { + dtor.future + } else { + return Some(try_local_did_span(did)); + }; + let def_key = tcx.def_key(dtor); + let Some(parent_index) = def_key.parent else { return Some(try_local_did_span(dtor)) }; + let parent_did = DefId { index: parent_index, krate: dtor.krate }; + Some(try_local_did_span(parent_did)) + } + ty::Coroutine(did, _) + | ty::CoroutineWitness(did, _) + | ty::CoroutineClosure(did, _) + | ty::Closure(did, _) + | ty::FnDef(did, _) + | ty::Foreign(did) => Some(tcx.def_span(did)), + ty::Param(_) => None, + } +} + +/// Check if a moved place at `idx` is a part of a BID. +/// The use of this check is that we will consider drops on these +/// as a drop of the overall BID and, thus, we can exclude it from the diagnosis. +fn place_descendent_of_bids<'tcx>( + mut idx: MovePathIndex, + move_data: &MoveData<'tcx>, + bids: &UnordSet<&Place<'tcx>>, +) -> bool { + loop { + let path = &move_data.move_paths[idx]; + if bids.contains(&path.place) { + return true; + } + if let Some(parent) = path.parent { + idx = parent; + } else { + return false; + } + } +} + +/// The core of the lint `tail-expr-drop-order` +pub(crate) fn run_lint<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId, body: &Body<'tcx>) { + if matches!(tcx.def_kind(def_id), rustc_hir::def::DefKind::SyntheticCoroutineBody) { + // A synthetic coroutine has no HIR body and it is enough to just analyse the original body + return; + } + if body.span.edition().at_least_rust_2024() + || tcx.lints_that_dont_need_to_run(()).contains(&lint::LintId::of(TAIL_EXPR_DROP_ORDER)) + { + return; + } + // ## About BIDs in blocks ## + // Track the set of blocks that contain a backwards-incompatible drop (BID) + // and, for each block, the vector of locations. + // + // We group them per-block because they tend to scheduled in the same drop ladder block. + let mut bid_per_block = IndexMap::default(); + let mut bid_places = UnordSet::new(); + let typing_env = ty::TypingEnv::post_analysis(tcx, def_id); + let mut ty_dropped_components = UnordMap::default(); + for (block, data) in body.basic_blocks.iter_enumerated() { + for (statement_index, stmt) in data.statements.iter().enumerate() { + if let StatementKind::BackwardIncompatibleDropHint { place, reason: _ } = &stmt.kind { + let ty = place.ty(body, tcx).ty; + if ty_dropped_components + .entry(ty) + .or_insert_with(|| extract_component_with_significant_dtor(tcx, typing_env, ty)) + .is_empty() + { + continue; + } + bid_per_block + .entry(block) + .or_insert(vec![]) + .push((Location { block, statement_index }, &**place)); + bid_places.insert(&**place); + } + } + } + if bid_per_block.is_empty() { + return; + } + + dump_mir(tcx, false, "lint_tail_expr_drop_order", &0 as _, body, |_, _| Ok(())); + let locals_with_user_names = collect_user_names(body); + let is_closure_like = tcx.is_closure_like(def_id.to_def_id()); + + // Compute the "maybe initialized" information for this body. + // When we encounter a DROP of some place P we only care + // about the drop if `P` may be initialized. + let move_data = MoveData::gather_moves(body, tcx, |_| true); + let maybe_init = MaybeInitializedPlaces::new(tcx, body, &move_data); + let mut maybe_init = maybe_init.iterate_to_fixpoint(tcx, body, None).into_results_cursor(body); + let mut block_drop_value_info = + IndexVec::from_elem_n(MovePathIndexAtBlock::Unknown, body.basic_blocks.len()); + for (&block, candidates) in &bid_per_block { + // We will collect drops on locals on paths between BID points to their actual drop locations + // into `all_locals_dropped`. + let mut all_locals_dropped = MixedBitSet::new_empty(move_data.move_paths.len()); + let mut drop_span = None; + for &(_, place) in candidates.iter() { + let mut collected_drops = MixedBitSet::new_empty(move_data.move_paths.len()); + // ## On detecting change in relative drop order ## + // Iterate through each BID-containing block `block`. + // If the place `P` targeted by the BID is "maybe initialized", + // then search forward to find the actual `DROP(P)` point. + // Everything dropped between the BID and the actual drop point + // is something whose relative drop order will change. + DropsReachable { + body, + place, + drop_span: &mut drop_span, + move_data: &move_data, + maybe_init: &mut maybe_init, + block_drop_value_info: &mut block_drop_value_info, + collected_drops: &mut collected_drops, + visited: Default::default(), + } + .visit(block); + // Compute the set `all_locals_dropped` of local variables that are dropped + // after the BID point but before the current drop point. + // + // These are the variables whose drop impls will be reordered with respect + // to `place`. + all_locals_dropped.union(&collected_drops); + } + + // We shall now exclude some local bindings for the following cases. + { + let mut to_exclude = MixedBitSet::new_empty(all_locals_dropped.domain_size()); + // We will now do subtraction from the candidate dropped locals, because of the + // following reasons. + for path_idx in all_locals_dropped.iter() { + let move_path = &move_data.move_paths[path_idx]; + let dropped_local = move_path.place.local; + // a) A return value _0 will eventually be used + // Example: + // fn f() -> Droppy { + // let _x = Droppy; + // Droppy + // } + // _0 holds the literal `Droppy` and rightfully `_x` has to be dropped first + if dropped_local == Local::ZERO { + debug!(?dropped_local, "skip return value"); + to_exclude.insert(path_idx); + continue; + } + // b) If we are analysing a closure, the captures are still dropped last. + // This is part of the closure capture lifetime contract. + // They are similar to the return value _0 with respect to lifetime rules. + if is_closure_like && matches!(dropped_local, ty::CAPTURE_STRUCT_LOCAL) { + debug!(?dropped_local, "skip closure captures"); + to_exclude.insert(path_idx); + continue; + } + // c) Sometimes we collect places that are projections into the BID locals, + // so they are considered dropped now. + // Example: + // struct NotVeryDroppy(Droppy); + // impl Drop for Droppy {..} + // fn f() -> NotVeryDroppy { + // let x = NotVeryDroppy(droppy()); + // { + // let y: Droppy = x.0; + // NotVeryDroppy(y) + // } + // } + // `y` takes `x.0`, which invalidates `x` as a complete `NotVeryDroppy` + // so there is no point in linting against `x` any more. + if place_descendent_of_bids(path_idx, &move_data, &bid_places) { + debug!(?dropped_local, "skip descendent of bids"); + to_exclude.insert(path_idx); + continue; + } + let observer_ty = move_path.place.ty(body, tcx).ty; + // d) The collected local has no custom destructor that passes our ecosystem filter. + if ty_dropped_components + .entry(observer_ty) + .or_insert_with(|| { + extract_component_with_significant_dtor(tcx, typing_env, observer_ty) + }) + .is_empty() + { + debug!(?dropped_local, "skip non-droppy types"); + to_exclude.insert(path_idx); + continue; + } + } + // Suppose that all BIDs point into the same local, + // we can remove the this local from the observed drops, + // so that we can focus our diagnosis more on the others. + if candidates.iter().all(|&(_, place)| candidates[0].1.local == place.local) { + for path_idx in all_locals_dropped.iter() { + if move_data.move_paths[path_idx].place.local == candidates[0].1.local { + to_exclude.insert(path_idx); + } + } + } + all_locals_dropped.subtract(&to_exclude); + } + if all_locals_dropped.is_empty() { + // No drop effect is observable, so let us move on. + continue; + } + + // ## The final work to assemble the diagnosis ## + // First collect or generate fresh names for local variable bindings and temporary values. + let local_names = assign_observables_names( + all_locals_dropped + .iter() + .map(|path_idx| move_data.move_paths[path_idx].place.local) + .chain(candidates.iter().map(|(_, place)| place.local)), + &locals_with_user_names, + ); + + let mut lint_root = None; + let mut local_labels = vec![]; + // We now collect the types with custom destructors. + for &(_, place) in candidates { + let linted_local_decl = &body.local_decls[place.local]; + let Some(&(ref name, is_generated_name)) = local_names.get(&place.local) else { + bug!("a name should have been assigned") + }; + let name = name.as_str(); + + if lint_root.is_none() + && let ClearCrossCrate::Set(data) = + &body.source_scopes[linted_local_decl.source_info.scope].local_data + { + lint_root = Some(data.lint_root); + } + + // Collect spans of the custom destructors. + let mut seen_dyn = false; + let destructors = ty_dropped_components + .get(&linted_local_decl.ty) + .unwrap() + .iter() + .filter_map(|&ty| { + if let Some(span) = ty_dtor_span(tcx, ty) { + Some(DestructorLabel { span, name, dtor_kind: "concrete" }) + } else if matches!(ty.kind(), ty::Dynamic(..)) { + if seen_dyn { + None + } else { + seen_dyn = true; + Some(DestructorLabel { span: DUMMY_SP, name, dtor_kind: "dyn" }) + } + } else { + None + } + }) + .collect(); + local_labels.push(LocalLabel { + span: linted_local_decl.source_info.span, + destructors, + name, + is_generated_name, + is_dropped_first_edition_2024: true, + }); + } + + // Similarly, custom destructors of the observed drops. + for path_idx in all_locals_dropped.iter() { + let place = &move_data.move_paths[path_idx].place; + // We are not using the type of the local because the drop may be partial. + let observer_ty = place.ty(body, tcx).ty; + + let observer_local_decl = &body.local_decls[place.local]; + let Some(&(ref name, is_generated_name)) = local_names.get(&place.local) else { + bug!("a name should have been assigned") + }; + let name = name.as_str(); + + let mut seen_dyn = false; + let destructors = extract_component_with_significant_dtor(tcx, typing_env, observer_ty) + .into_iter() + .filter_map(|ty| { + if let Some(span) = ty_dtor_span(tcx, ty) { + Some(DestructorLabel { span, name, dtor_kind: "concrete" }) + } else if matches!(ty.kind(), ty::Dynamic(..)) { + if seen_dyn { + None + } else { + seen_dyn = true; + Some(DestructorLabel { span: DUMMY_SP, name, dtor_kind: "dyn" }) + } + } else { + None + } + }) + .collect(); + local_labels.push(LocalLabel { + span: observer_local_decl.source_info.span, + destructors, + name, + is_generated_name, + is_dropped_first_edition_2024: false, + }); + } + + let span = local_labels[0].span; + tcx.emit_node_span_lint( + lint::builtin::TAIL_EXPR_DROP_ORDER, + lint_root.unwrap_or(CRATE_HIR_ID), + span, + TailExprDropOrderLint { local_labels, drop_span, _epilogue: () }, + ); + } +} + +/// Extract binding names if available for diagnosis +fn collect_user_names(body: &Body<'_>) -> IndexMap<Local, Symbol> { + let mut names = IndexMap::default(); + for var_debug_info in &body.var_debug_info { + if let mir::VarDebugInfoContents::Place(place) = &var_debug_info.value + && let Some(local) = place.local_or_deref_local() + { + names.entry(local).or_insert(var_debug_info.name); + } + } + names +} + +/// Assign names for anonymous or temporary values for diagnosis +fn assign_observables_names( + locals: impl IntoIterator<Item = Local>, + user_names: &IndexMap<Local, Symbol>, +) -> IndexMap<Local, (String, bool)> { + let mut names = IndexMap::default(); + let mut assigned_names = FxHashSet::default(); + let mut idx = 0u64; + let mut fresh_name = || { + idx += 1; + (format!("#{idx}"), true) + }; + for local in locals { + let name = if let Some(name) = user_names.get(&local) { + let name = name.as_str(); + if assigned_names.contains(name) { fresh_name() } else { (name.to_owned(), false) } + } else { + fresh_name() + }; + assigned_names.insert(name.0.clone()); + names.insert(local, name); + } + names +} + +#[derive(LintDiagnostic)] +#[diag(mir_transform_tail_expr_drop_order)] +struct TailExprDropOrderLint<'a> { + #[subdiagnostic] + local_labels: Vec<LocalLabel<'a>>, + #[label(mir_transform_drop_location)] + drop_span: Option<Span>, + #[note(mir_transform_note_epilogue)] + _epilogue: (), +} + +struct LocalLabel<'a> { + span: Span, + name: &'a str, + is_generated_name: bool, + is_dropped_first_edition_2024: bool, + destructors: Vec<DestructorLabel<'a>>, +} + +/// A custom `Subdiagnostic` implementation so that the notes are delivered in a specific order +impl Subdiagnostic for LocalLabel<'_> { + fn add_to_diag_with< + G: rustc_errors::EmissionGuarantee, + F: rustc_errors::SubdiagMessageOp<G>, + >( + self, + diag: &mut rustc_errors::Diag<'_, G>, + f: &F, + ) { + diag.arg("name", self.name); + diag.arg("is_generated_name", self.is_generated_name); + diag.arg("is_dropped_first_edition_2024", self.is_dropped_first_edition_2024); + let msg = f(diag, crate::fluent_generated::mir_transform_tail_expr_local.into()); + diag.span_label(self.span, msg); + for dtor in self.destructors { + dtor.add_to_diag_with(diag, f); + } + let msg = f(diag, crate::fluent_generated::mir_transform_label_local_epilogue.into()); + diag.span_label(self.span, msg); + } +} + +#[derive(Subdiagnostic)] +#[note(mir_transform_tail_expr_dtor)] +struct DestructorLabel<'a> { + #[primary_span] + span: Span, + dtor_kind: &'static str, + name: &'a str, +} diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs index 6d635606687..942c6144ea6 100644 --- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs +++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs @@ -3,7 +3,7 @@ use rustc_middle::mir::*; use rustc_middle::ty::{self, TyCtxt}; use rustc_middle::{bug, span_bug}; -use rustc_span::symbol::sym; +use rustc_span::sym; use crate::take_array; diff --git a/compiler/rustc_mir_transform/src/match_branches.rs b/compiler/rustc_mir_transform/src/match_branches.rs index 237227f5294..20e2a65b311 100644 --- a/compiler/rustc_mir_transform/src/match_branches.rs +++ b/compiler/rustc_mir_transform/src/match_branches.rs @@ -5,7 +5,7 @@ use rustc_index::IndexSlice; use rustc_middle::mir::patch::MirPatch; use rustc_middle::mir::*; use rustc_middle::ty::layout::{IntegerExt, TyAndLayout}; -use rustc_middle::ty::{ParamEnv, ScalarInt, Ty, TyCtxt}; +use rustc_middle::ty::{self, ScalarInt, Ty, TyCtxt}; use rustc_type_ir::TyKind::*; use super::simplify::simplify_cfg; @@ -18,17 +18,11 @@ impl<'tcx> crate::MirPass<'tcx> for MatchBranchSimplification { } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let def_id = body.source.def_id(); - let param_env = tcx.param_env_reveal_all_normalized(def_id); - + let typing_env = body.typing_env(tcx); let mut should_cleanup = false; for i in 0..body.basic_blocks.len() { let bbs = &*body.basic_blocks; let bb_idx = BasicBlock::from_usize(i); - if !tcx.consider_optimizing(|| format!("MatchBranchSimplification {def_id:?} ")) { - continue; - } - match bbs[bb_idx].terminator().kind { TerminatorKind::SwitchInt { discr: ref _discr @ (Operand::Copy(_) | Operand::Move(_)), @@ -40,11 +34,11 @@ impl<'tcx> crate::MirPass<'tcx> for MatchBranchSimplification { _ => continue, }; - if SimplifyToIf.simplify(tcx, body, bb_idx, param_env).is_some() { + if SimplifyToIf.simplify(tcx, body, bb_idx, typing_env).is_some() { should_cleanup = true; continue; } - if SimplifyToExp::default().simplify(tcx, body, bb_idx, param_env).is_some() { + if SimplifyToExp::default().simplify(tcx, body, bb_idx, typing_env).is_some() { should_cleanup = true; continue; } @@ -65,7 +59,7 @@ trait SimplifyMatch<'tcx> { tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, switch_bb_idx: BasicBlock, - param_env: ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, ) -> Option<()> { let bbs = &body.basic_blocks; let (discr, targets) = match bbs[switch_bb_idx].terminator().kind { @@ -74,7 +68,7 @@ trait SimplifyMatch<'tcx> { }; let discr_ty = discr.ty(body.local_decls(), tcx); - self.can_simplify(tcx, targets, param_env, bbs, discr_ty)?; + self.can_simplify(tcx, targets, typing_env, bbs, discr_ty)?; let mut patch = MirPatch::new(body); @@ -90,7 +84,16 @@ trait SimplifyMatch<'tcx> { let parent_end = Location { block: switch_bb_idx, statement_index }; patch.add_statement(parent_end, StatementKind::StorageLive(discr_local)); patch.add_assign(parent_end, Place::from(discr_local), Rvalue::Use(discr)); - self.new_stmts(tcx, targets, param_env, &mut patch, parent_end, bbs, discr_local, discr_ty); + self.new_stmts( + tcx, + targets, + typing_env, + &mut patch, + parent_end, + bbs, + discr_local, + discr_ty, + ); patch.add_statement(parent_end, StatementKind::StorageDead(discr_local)); patch.patch_terminator(switch_bb_idx, bbs[first].terminator().kind.clone()); patch.apply(body); @@ -104,7 +107,7 @@ trait SimplifyMatch<'tcx> { &mut self, tcx: TyCtxt<'tcx>, targets: &SwitchTargets, - param_env: ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, bbs: &IndexSlice<BasicBlock, BasicBlockData<'tcx>>, discr_ty: Ty<'tcx>, ) -> Option<()>; @@ -113,7 +116,7 @@ trait SimplifyMatch<'tcx> { &self, tcx: TyCtxt<'tcx>, targets: &SwitchTargets, - param_env: ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, patch: &mut MirPatch<'tcx>, parent_end: Location, bbs: &IndexSlice<BasicBlock, BasicBlockData<'tcx>>, @@ -160,7 +163,7 @@ impl<'tcx> SimplifyMatch<'tcx> for SimplifyToIf { &mut self, tcx: TyCtxt<'tcx>, targets: &SwitchTargets, - param_env: ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, bbs: &IndexSlice<BasicBlock, BasicBlockData<'tcx>>, _discr_ty: Ty<'tcx>, ) -> Option<()> { @@ -197,8 +200,8 @@ impl<'tcx> SimplifyMatch<'tcx> for SimplifyToIf { ) if lhs_f == lhs_s && f_c.const_.ty().is_bool() && s_c.const_.ty().is_bool() - && f_c.const_.try_eval_bool(tcx, param_env).is_some() - && s_c.const_.try_eval_bool(tcx, param_env).is_some() => {} + && f_c.const_.try_eval_bool(tcx, typing_env).is_some() + && s_c.const_.try_eval_bool(tcx, typing_env).is_some() => {} // Otherwise we cannot optimize. Try another block. _ => return None, @@ -211,7 +214,7 @@ impl<'tcx> SimplifyMatch<'tcx> for SimplifyToIf { &self, tcx: TyCtxt<'tcx>, targets: &SwitchTargets, - param_env: ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, patch: &mut MirPatch<'tcx>, parent_end: Location, bbs: &IndexSlice<BasicBlock, BasicBlockData<'tcx>>, @@ -235,15 +238,15 @@ impl<'tcx> SimplifyMatch<'tcx> for SimplifyToIf { StatementKind::Assign(box (_, Rvalue::Use(Operand::Constant(s_c)))), ) => { // From earlier loop we know that we are dealing with bool constants only: - let f_b = f_c.const_.try_eval_bool(tcx, param_env).unwrap(); - let s_b = s_c.const_.try_eval_bool(tcx, param_env).unwrap(); + let f_b = f_c.const_.try_eval_bool(tcx, typing_env).unwrap(); + let s_b = s_c.const_.try_eval_bool(tcx, typing_env).unwrap(); if f_b == s_b { // Same value in both blocks. Use statement as is. patch.add_statement(parent_end, f.kind.clone()); } else { // Different value between blocks. Make value conditional on switch // condition. - let size = tcx.layout_of(param_env.and(discr_ty)).unwrap().size; + let size = tcx.layout_of(typing_env.as_query_input(discr_ty)).unwrap().size; let const_cmp = Operand::const_from_scalar( tcx, discr_ty, @@ -363,7 +366,7 @@ impl<'tcx> SimplifyMatch<'tcx> for SimplifyToExp { &mut self, tcx: TyCtxt<'tcx>, targets: &SwitchTargets, - param_env: ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, bbs: &IndexSlice<BasicBlock, BasicBlockData<'tcx>>, discr_ty: Ty<'tcx>, ) -> Option<()> { @@ -388,7 +391,7 @@ impl<'tcx> SimplifyMatch<'tcx> for SimplifyToExp { return None; } - let discr_layout = tcx.layout_of(param_env.and(discr_ty)).unwrap(); + let discr_layout = tcx.layout_of(typing_env.as_query_input(discr_ty)).unwrap(); let first_stmts = &bbs[first_target].statements; let (second_case_val, second_target) = target_iter.next().unwrap(); let second_stmts = &bbs[second_target].statements; @@ -414,8 +417,8 @@ impl<'tcx> SimplifyMatch<'tcx> for SimplifyToExp { && f_c.const_.ty().is_integral() => { match ( - f_c.const_.try_eval_scalar_int(tcx, param_env), - s_c.const_.try_eval_scalar_int(tcx, param_env), + f_c.const_.try_eval_scalar_int(tcx, typing_env), + s_c.const_.try_eval_scalar_int(tcx, typing_env), ) { (Some(f), Some(s)) if f == s => ExpectedTransformKind::SameByEq { place: lhs_f, @@ -467,11 +470,11 @@ impl<'tcx> SimplifyMatch<'tcx> for SimplifyToExp { StatementKind::Assign(box (lhs_s, Rvalue::Use(Operand::Constant(s_c)))), ) if lhs_f == lhs_s && s_c.const_.ty() == f_ty - && s_c.const_.try_eval_scalar_int(tcx, param_env) == Some(scalar) => {} + && s_c.const_.try_eval_scalar_int(tcx, typing_env) == Some(scalar) => {} ( ExpectedTransformKind::Cast { place: lhs_f, ty: f_ty }, StatementKind::Assign(box (lhs_s, Rvalue::Use(Operand::Constant(s_c)))), - ) if let Some(f) = s_c.const_.try_eval_scalar_int(tcx, param_env) + ) if let Some(f) = s_c.const_.try_eval_scalar_int(tcx, typing_env) && lhs_f == lhs_s && s_c.const_.ty() == f_ty && can_cast(tcx, other_val, discr_layout, f_ty, f) => {} @@ -487,7 +490,7 @@ impl<'tcx> SimplifyMatch<'tcx> for SimplifyToExp { &self, _tcx: TyCtxt<'tcx>, targets: &SwitchTargets, - _param_env: ParamEnv<'tcx>, + _typing_env: ty::TypingEnv<'tcx>, patch: &mut MirPatch<'tcx>, parent_end: Location, bbs: &IndexSlice<BasicBlock, BasicBlockData<'tcx>>, diff --git a/compiler/rustc_mir_transform/src/multiple_return_terminators.rs b/compiler/rustc_mir_transform/src/multiple_return_terminators.rs index b6d6ef5de1d..a9227524ce5 100644 --- a/compiler/rustc_mir_transform/src/multiple_return_terminators.rs +++ b/compiler/rustc_mir_transform/src/multiple_return_terminators.rs @@ -14,10 +14,9 @@ impl<'tcx> crate::MirPass<'tcx> for MultipleReturnTerminators { sess.mir_opt_level() >= 4 } - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + fn run_pass(&self, _: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // find basic blocks with no statement and a return terminator let mut bbs_simple_returns = BitSet::new_empty(body.basic_blocks.len()); - let def_id = body.source.def_id(); let bbs = body.basic_blocks_mut(); for idx in bbs.indices() { if bbs[idx].statements.is_empty() @@ -28,10 +27,6 @@ impl<'tcx> crate::MirPass<'tcx> for MultipleReturnTerminators { } for bb in bbs { - if !tcx.consider_optimizing(|| format!("MultipleReturnTerminators {def_id:?} ")) { - break; - } - if let TerminatorKind::Goto { target } = bb.terminator().kind { if bbs_simple_returns.contains(target) { bb.terminator_mut().kind = TerminatorKind::Return; diff --git a/compiler/rustc_mir_transform/src/nrvo.rs b/compiler/rustc_mir_transform/src/nrvo.rs index 98fa149e2bc..cd026ed6806 100644 --- a/compiler/rustc_mir_transform/src/nrvo.rs +++ b/compiler/rustc_mir_transform/src/nrvo.rs @@ -45,10 +45,6 @@ impl<'tcx> crate::MirPass<'tcx> for RenameReturnPlace { return; }; - if !tcx.consider_optimizing(|| format!("RenameReturnPlace {def_id:?}")) { - return; - } - debug!( "`{:?}` was eligible for NRVO, making {:?} the return place", def_id, returned_local diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs index 29f8b4f6e4d..8a45ce0762d 100644 --- a/compiler/rustc_mir_transform/src/pass_manager.rs +++ b/compiler/rustc_mir_transform/src/pass_manager.rs @@ -1,24 +1,25 @@ use std::cell::RefCell; use std::collections::hash_map::Entry; -use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::fx::{FxHashMap, FxIndexSet}; use rustc_middle::mir::{self, Body, MirPhase, RuntimePhase}; use rustc_middle::ty::TyCtxt; use rustc_session::Session; use tracing::trace; use crate::lint::lint_body; -use crate::validate; +use crate::{errors, validate}; thread_local! { - static PASS_NAMES: RefCell<FxHashMap<&'static str, &'static str>> = { + /// Maps MIR pass names to a snake case form to match profiling naming style + static PASS_TO_PROFILER_NAMES: RefCell<FxHashMap<&'static str, &'static str>> = { RefCell::new(FxHashMap::default()) }; } /// Converts a MIR pass name into a snake case form to match the profiling naming style. fn to_profiler_name(type_name: &'static str) -> &'static str { - PASS_NAMES.with(|names| match names.borrow_mut().entry(type_name) { + PASS_TO_PROFILER_NAMES.with(|names| match names.borrow_mut().entry(type_name) { Entry::Occupied(e) => *e.get(), Entry::Vacant(e) => { let snake_case: String = type_name @@ -198,6 +199,31 @@ fn run_passes_inner<'tcx>( let overridden_passes = &tcx.sess.opts.unstable_opts.mir_enable_passes; trace!(?overridden_passes); + let named_passes: FxIndexSet<_> = + overridden_passes.iter().map(|(name, _)| name.as_str()).collect(); + + for &name in named_passes.difference(&*crate::PASS_NAMES) { + tcx.dcx().emit_warn(errors::UnknownPassName { name }); + } + + // Verify that no passes are missing from the `declare_passes` invocation + #[cfg(debug_assertions)] + #[allow(rustc::diagnostic_outside_of_impl)] + #[allow(rustc::untranslatable_diagnostic)] + { + let used_passes: FxIndexSet<_> = passes.iter().map(|p| p.name()).collect(); + + let undeclared = used_passes.difference(&*crate::PASS_NAMES).collect::<Vec<_>>(); + if let Some((name, rest)) = undeclared.split_first() { + let mut err = + tcx.dcx().struct_bug(format!("pass `{name}` is not declared in `PASS_NAMES`")); + for name in rest { + err.note(format!("pass `{name}` is also not declared in `PASS_NAMES`")); + } + err.emit(); + } + } + let prof_arg = tcx.sess.prof.enabled().then(|| format!("{:?}", body.source.def_id())); if !body.should_skip() { @@ -266,7 +292,7 @@ fn run_passes_inner<'tcx>( } pub(super) fn validate_body<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, when: String) { - validate::Validator { when, mir_phase: body.phase }.run_pass(tcx, body); + validate::Validator { when }.run_pass(tcx, body); } fn dump_mir_for_pass<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, pass_name: &str, is_after: bool) { diff --git a/compiler/rustc_mir_transform/src/reveal_all.rs b/compiler/rustc_mir_transform/src/post_analysis_normalize.rs index f3b2f78b31c..3eecf79a7ea 100644 --- a/compiler/rustc_mir_transform/src/reveal_all.rs +++ b/compiler/rustc_mir_transform/src/post_analysis_normalize.rs @@ -1,24 +1,28 @@ -//! Normalizes MIR in RevealAll mode. +//! Normalizes MIR in `TypingMode::PostAnalysis` mode, most notably revealing +//! its opaques. We also only normalize specializable associated items once in +//! `PostAnalysis` mode. use rustc_middle::mir::visit::*; use rustc_middle::mir::*; use rustc_middle::ty::{self, Ty, TyCtxt}; -pub(super) struct RevealAll; +pub(super) struct PostAnalysisNormalize; -impl<'tcx> crate::MirPass<'tcx> for RevealAll { +impl<'tcx> crate::MirPass<'tcx> for PostAnalysisNormalize { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); - RevealAllVisitor { tcx, param_env }.visit_body_preserves_cfg(body); + // FIXME(#132279): This is used during the phase transition from analysis + // to runtime, so we have to manually specify the correct typing mode. + let typing_env = ty::TypingEnv::post_analysis(tcx, body.source.def_id()); + PostAnalysisNormalizeVisitor { tcx, typing_env }.visit_body_preserves_cfg(body); } } -struct RevealAllVisitor<'tcx> { +struct PostAnalysisNormalizeVisitor<'tcx> { tcx: TyCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, } -impl<'tcx> MutVisitor<'tcx> for RevealAllVisitor<'tcx> { +impl<'tcx> MutVisitor<'tcx> for PostAnalysisNormalizeVisitor<'tcx> { #[inline] fn tcx(&self) -> TyCtxt<'tcx> { self.tcx @@ -36,7 +40,7 @@ impl<'tcx> MutVisitor<'tcx> for RevealAllVisitor<'tcx> { return; } // `OpaqueCast` projections are only needed if there are opaque types on which projections - // are performed. After the `RevealAll` pass, all opaque types are replaced with their + // are performed. After the `PostAnalysisNormalize` pass, all opaque types are replaced with their // hidden types, so we don't need these projections anymore. place.projection = self.tcx.mk_place_elems( &place @@ -53,7 +57,7 @@ impl<'tcx> MutVisitor<'tcx> for RevealAllVisitor<'tcx> { // We have to use `try_normalize_erasing_regions` here, since it's // possible that we visit impossible-to-satisfy where clauses here, // see #91745 - if let Ok(c) = self.tcx.try_normalize_erasing_regions(self.param_env, constant.const_) { + if let Ok(c) = self.tcx.try_normalize_erasing_regions(self.typing_env, constant.const_) { constant.const_ = c; } self.super_const_operand(constant, location); @@ -64,7 +68,7 @@ impl<'tcx> MutVisitor<'tcx> for RevealAllVisitor<'tcx> { // We have to use `try_normalize_erasing_regions` here, since it's // possible that we visit impossible-to-satisfy where clauses here, // see #91745 - if let Ok(t) = self.tcx.try_normalize_erasing_regions(self.param_env, *ty) { + if let Ok(t) = self.tcx.try_normalize_erasing_regions(self.typing_env, *ty) { *ty = t; } } diff --git a/compiler/rustc_mir_transform/src/promote_consts.rs b/compiler/rustc_mir_transform/src/promote_consts.rs index fa9a6bfcf7c..6be95b1f0f1 100644 --- a/compiler/rustc_mir_transform/src/promote_consts.rs +++ b/compiler/rustc_mir_transform/src/promote_consts.rs @@ -325,7 +325,7 @@ impl<'tcx> Validator<'_, 'tcx> { if let TempState::Defined { location: loc, .. } = self.temps[local] && let Left(statement) = self.body.stmt_at(loc) && let Some((_, Rvalue::Use(Operand::Constant(c)))) = statement.kind.as_assign() - && let Some(idx) = c.const_.try_eval_target_usize(self.tcx, self.param_env) + && let Some(idx) = c.const_.try_eval_target_usize(self.tcx, self.typing_env) // Determine the type of the thing we are indexing. && let ty::Array(_, len) = place_base.ty(self.body, self.tcx).ty.kind() // It's an array; determine its length. @@ -490,7 +490,7 @@ impl<'tcx> Validator<'_, 'tcx> { // Integer division: the RHS must be a non-zero const. let rhs_val = match rhs { Operand::Constant(c) => { - c.const_.try_eval_scalar_int(self.tcx, self.param_env) + c.const_.try_eval_scalar_int(self.tcx, self.typing_env) } _ => None, }; @@ -509,7 +509,7 @@ impl<'tcx> Validator<'_, 'tcx> { let lhs_val = match lhs { Operand::Constant(c) => c .const_ - .try_eval_scalar_int(self.tcx, self.param_env), + .try_eval_scalar_int(self.tcx, self.typing_env), _ => None, }; let lhs_min = sz.signed_int_min(); diff --git a/compiler/rustc_mir_transform/src/ref_prop.rs b/compiler/rustc_mir_transform/src/ref_prop.rs index b11b503e8d4..96bcdfa6fac 100644 --- a/compiler/rustc_mir_transform/src/ref_prop.rs +++ b/compiler/rustc_mir_transform/src/ref_prop.rs @@ -8,8 +8,7 @@ use rustc_middle::mir::visit::*; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; use rustc_mir_dataflow::Analysis; -use rustc_mir_dataflow::impls::MaybeStorageDead; -use rustc_mir_dataflow::storage::always_storage_live_locals; +use rustc_mir_dataflow::impls::{MaybeStorageDead, always_storage_live_locals}; use tracing::{debug, instrument}; use crate::ssa::{SsaLocals, StorageLiveLocals}; @@ -85,8 +84,8 @@ impl<'tcx> crate::MirPass<'tcx> for ReferencePropagation { } fn propagate_ssa<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { - let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); - let ssa = SsaLocals::new(tcx, body, param_env); + let typing_env = body.typing_env(tcx); + let ssa = SsaLocals::new(tcx, body, typing_env); let mut replacer = compute_replacement(tcx, body, &ssa); debug!(?replacer.targets); diff --git a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs index 55394e93a5c..fd49e956f43 100644 --- a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs +++ b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs @@ -92,6 +92,7 @@ impl RemoveNoopLandingPads { | StatementKind::AscribeUserType(..) | StatementKind::Coverage(..) | StatementKind::ConstEvalCounter + | StatementKind::BackwardIncompatibleDropHint { .. } | StatementKind::Nop => { // These are all noops in a landing pad } diff --git a/compiler/rustc_mir_transform/src/remove_uninit_drops.rs b/compiler/rustc_mir_transform/src/remove_uninit_drops.rs index 20c34a7469e..e955d8277a4 100644 --- a/compiler/rustc_mir_transform/src/remove_uninit_drops.rs +++ b/compiler/rustc_mir_transform/src/remove_uninit_drops.rs @@ -1,7 +1,7 @@ use rustc_abi::FieldIdx; -use rustc_index::bit_set::ChunkedBitSet; +use rustc_index::bit_set::MixedBitSet; use rustc_middle::mir::{Body, TerminatorKind}; -use rustc_middle::ty::{self, GenericArgsRef, ParamEnv, Ty, TyCtxt, VariantDef}; +use rustc_middle::ty::{self, GenericArgsRef, Ty, TyCtxt, VariantDef}; use rustc_mir_dataflow::impls::MaybeInitializedPlaces; use rustc_mir_dataflow::move_paths::{LookupResult, MoveData, MovePathIndex}; use rustc_mir_dataflow::{Analysis, MaybeReachable, move_path_children_matching}; @@ -18,8 +18,8 @@ pub(super) struct RemoveUninitDrops; impl<'tcx> crate::MirPass<'tcx> for RemoveUninitDrops { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let param_env = tcx.param_env(body.source.def_id()); - let move_data = MoveData::gather_moves(body, tcx, |ty| ty.needs_drop(tcx, param_env)); + let typing_env = body.typing_env(tcx); + let move_data = MoveData::gather_moves(body, tcx, |ty| ty.needs_drop(tcx, typing_env)); let mut maybe_inits = MaybeInitializedPlaces::new(tcx, body, &move_data) .iterate_to_fixpoint(tcx, body, Some("remove_uninit_drops")) @@ -40,7 +40,7 @@ impl<'tcx> crate::MirPass<'tcx> for RemoveUninitDrops { let should_keep = is_needs_drop_and_init( tcx, - param_env, + typing_env, maybe_inits, &move_data, place.ty(body, tcx).ty, @@ -66,24 +66,24 @@ impl<'tcx> crate::MirPass<'tcx> for RemoveUninitDrops { fn is_needs_drop_and_init<'tcx>( tcx: TyCtxt<'tcx>, - param_env: ParamEnv<'tcx>, - maybe_inits: &ChunkedBitSet<MovePathIndex>, + typing_env: ty::TypingEnv<'tcx>, + maybe_inits: &MixedBitSet<MovePathIndex>, move_data: &MoveData<'tcx>, ty: Ty<'tcx>, mpi: MovePathIndex, ) -> bool { // No need to look deeper if the root is definitely uninit or if it has no `Drop` impl. - if !maybe_inits.contains(mpi) || !ty.needs_drop(tcx, param_env) { + if !maybe_inits.contains(mpi) || !ty.needs_drop(tcx, typing_env) { return false; } let field_needs_drop_and_init = |(f, f_ty, mpi)| { let child = move_path_children_matching(move_data, mpi, |x| x.is_field_to(f)); let Some(mpi) = child else { - return Ty::needs_drop(f_ty, tcx, param_env); + return Ty::needs_drop(f_ty, tcx, typing_env); }; - is_needs_drop_and_init(tcx, param_env, maybe_inits, move_data, f_ty, mpi) + is_needs_drop_and_init(tcx, typing_env, maybe_inits, move_data, f_ty, mpi) }; // This pass is only needed for const-checking, so it doesn't handle as many cases as @@ -110,7 +110,7 @@ fn is_needs_drop_and_init<'tcx>( let downcast = move_path_children_matching(move_data, mpi, |x| x.is_downcast_to(vid)); let Some(dc_mpi) = downcast else { - return variant_needs_drop(tcx, param_env, args, variant); + return variant_needs_drop(tcx, typing_env, args, variant); }; dc_mpi @@ -139,12 +139,12 @@ fn is_needs_drop_and_init<'tcx>( fn variant_needs_drop<'tcx>( tcx: TyCtxt<'tcx>, - param_env: ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, args: GenericArgsRef<'tcx>, variant: &VariantDef, ) -> bool { variant.fields.iter().any(|field| { let f_ty = field.ty(tcx, args); - f_ty.needs_drop(tcx, param_env) + f_ty.needs_drop(tcx, typing_env) }) } diff --git a/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs b/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs index 28925ba1beb..e335051d656 100644 --- a/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs +++ b/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs @@ -2,7 +2,8 @@ //! //! When the MIR is built, we check `needs_drop` before emitting a `Drop` for a place. This pass is //! useful because (unlike MIR building) it runs after type checking, so it can make use of -//! `Reveal::All` to provide more precise type information. +//! `TypingMode::PostAnalysis` to provide more precise type information, especially about opaque +//! types. use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; @@ -16,18 +17,13 @@ impl<'tcx> crate::MirPass<'tcx> for RemoveUnneededDrops { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { trace!("Running RemoveUnneededDrops on {:?}", body.source); - let did = body.source.def_id(); - let param_env = tcx.param_env_reveal_all_normalized(did); + let typing_env = body.typing_env(tcx); let mut should_simplify = false; - for block in body.basic_blocks.as_mut() { let terminator = block.terminator_mut(); if let TerminatorKind::Drop { place, target, .. } = terminator.kind { let ty = place.ty(&body.local_decls, tcx); - if ty.ty.needs_drop(tcx, param_env) { - continue; - } - if !tcx.consider_optimizing(|| format!("RemoveUnneededDrops {did:?} ")) { + if ty.ty.needs_drop(tcx, typing_env) { continue; } debug!("SUCCESS: replacing `drop` with goto({:?})", target); diff --git a/compiler/rustc_mir_transform/src/remove_zsts.rs b/compiler/rustc_mir_transform/src/remove_zsts.rs index f13bb1c5993..64e183bcbc0 100644 --- a/compiler/rustc_mir_transform/src/remove_zsts.rs +++ b/compiler/rustc_mir_transform/src/remove_zsts.rs @@ -17,13 +17,9 @@ impl<'tcx> crate::MirPass<'tcx> for RemoveZsts { return; } - if !tcx.consider_optimizing(|| format!("RemoveZsts - {:?}", body.source.def_id())) { - return; - } - - let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); + let typing_env = body.typing_env(tcx); let local_decls = &body.local_decls; - let mut replacer = Replacer { tcx, param_env, local_decls }; + let mut replacer = Replacer { tcx, typing_env, local_decls }; for var_debug_info in &mut body.var_debug_info { replacer.visit_var_debug_info(var_debug_info); } @@ -35,7 +31,7 @@ impl<'tcx> crate::MirPass<'tcx> for RemoveZsts { struct Replacer<'a, 'tcx> { tcx: TyCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, local_decls: &'a LocalDecls<'tcx>, } @@ -61,7 +57,7 @@ impl<'tcx> Replacer<'_, 'tcx> { if !maybe_zst(ty) { return false; } - let Ok(layout) = self.tcx.layout_of(self.param_env.and(ty)) else { + let Ok(layout) = self.tcx.layout_of(self.typing_env.as_query_input(ty)) else { return false; }; layout.is_zst() @@ -94,16 +90,12 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'_, 'tcx> { } } - fn visit_operand(&mut self, operand: &mut Operand<'tcx>, loc: Location) { + fn visit_operand(&mut self, operand: &mut Operand<'tcx>, _: Location) { if let Operand::Constant(_) = operand { return; } let op_ty = operand.ty(self.local_decls, self.tcx); - if self.known_to_be_zst(op_ty) - && self.tcx.consider_optimizing(|| { - format!("RemoveZsts - Operand: {operand:?} Location: {loc:?}") - }) - { + if self.known_to_be_zst(op_ty) { *operand = Operand::Constant(Box::new(self.make_zst(op_ty))) } } @@ -125,6 +117,7 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'_, 'tcx> { StatementKind::Coverage(_) | StatementKind::Intrinsic(_) | StatementKind::Nop + | StatementKind::BackwardIncompatibleDropHint { .. } | StatementKind::ConstEvalCounter => None, }; if let Some(place_for_ty) = place_for_ty diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index ffa11f5b213..722da3c420d 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -9,6 +9,7 @@ use rustc_index::{Idx, IndexVec}; use rustc_middle::mir::patch::MirPatch; use rustc_middle::mir::*; use rustc_middle::query::Providers; +use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::{ self, CoroutineArgs, CoroutineArgsExt, EarlyBinder, GenericArgs, Ty, TyCtxt, }; @@ -141,7 +142,7 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body< debug!("make_shim({:?}) = untransformed {:?}", instance, result); // We don't validate MIR here because the shims may generate code that's - // only valid in a reveal-all param-env. However, since we do initial + // only valid in a `PostAnalysis` param-env. However, since we do initial // validation with the MirBuilt phase, which uses a user-facing param-env. // This causes validation errors when TAITs are involved. pm::run_passes_no_validate( @@ -274,9 +275,9 @@ fn build_drop_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, ty: Option<Ty<'tcx>>) if ty.is_some() { let patch = { - let param_env = tcx.param_env_reveal_all_normalized(def_id); + let typing_env = ty::TypingEnv::post_analysis(tcx, def_id); let mut elaborator = - DropShimElaborator { body: &body, patch: MirPatch::new(&body), tcx, param_env }; + DropShimElaborator { body: &body, patch: MirPatch::new(&body), tcx, typing_env }; let dropee = tcx.mk_place_deref(dropee_ptr); let resume_block = elaborator.patch.resume_block(); elaborate_drops::elaborate_drop( @@ -334,7 +335,7 @@ pub(super) struct DropShimElaborator<'a, 'tcx> { pub body: &'a Body<'tcx>, pub patch: MirPatch<'tcx>, pub tcx: TyCtxt<'tcx>, - pub param_env: ty::ParamEnv<'tcx>, + pub typing_env: ty::TypingEnv<'tcx>, } impl fmt::Debug for DropShimElaborator<'_, '_> { @@ -355,8 +356,8 @@ impl<'a, 'tcx> DropElaborator<'a, 'tcx> for DropShimElaborator<'a, 'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } - fn param_env(&self) -> ty::ParamEnv<'tcx> { - self.param_env + fn typing_env(&self) -> ty::TypingEnv<'tcx> { + self.typing_env } fn drop_style(&self, _path: Self::Path, mode: DropFlagMode) -> DropStyle { @@ -710,6 +711,13 @@ fn build_call_shim<'tcx>( }; let def_id = instance.def_id(); + + let rpitit_shim = if let ty::InstanceKind::ReifyShim(..) = instance { + tcx.return_position_impl_trait_in_trait_shim_data(def_id) + } else { + None + }; + let sig = tcx.fn_sig(def_id); let sig = sig.map_bound(|sig| tcx.instantiate_bound_regions_with_erased(sig)); @@ -747,8 +755,8 @@ fn build_call_shim<'tcx>( sig.inputs_and_output = tcx.mk_type_list(&inputs_and_output); } - // FIXME(eddyb) avoid having this snippet both here and in - // `Instance::fn_sig` (introduce `InstanceKind::fn_sig`?). + // FIXME: Avoid having to adjust the signature both here and in + // `fn_sig_for_fn_abi`. if let ty::InstanceKind::VTableShim(..) = instance { // Modify fn(self, ...) to fn(self: *mut Self, ...) let mut inputs_and_output = sig.inputs_and_output.to_vec(); @@ -765,9 +773,34 @@ fn build_call_shim<'tcx>( let mut local_decls = local_decls_for_sig(&sig, span); let source_info = SourceInfo::outermost(span); + let mut destination = Place::return_place(); + if let Some((rpitit_def_id, fn_args)) = rpitit_shim { + let rpitit_args = + fn_args.instantiate_identity().extend_to(tcx, rpitit_def_id, |param, _| { + match param.kind { + ty::GenericParamDefKind::Lifetime => tcx.lifetimes.re_erased.into(), + ty::GenericParamDefKind::Type { .. } + | ty::GenericParamDefKind::Const { .. } => { + unreachable!("rpitit should have no addition ty/ct") + } + } + }); + let dyn_star_ty = Ty::new_dynamic( + tcx, + tcx.item_bounds_to_existential_predicates(rpitit_def_id, rpitit_args), + tcx.lifetimes.re_erased, + ty::DynStar, + ); + destination = local_decls.push(local_decls[RETURN_PLACE].clone()).into(); + local_decls[RETURN_PLACE].ty = dyn_star_ty; + let mut inputs_and_output = sig.inputs_and_output.to_vec(); + *inputs_and_output.last_mut().unwrap() = dyn_star_ty; + sig.inputs_and_output = tcx.mk_type_list(&inputs_and_output); + } + let rcvr_place = || { assert!(rcvr_adjustment.is_some()); - Place::from(Local::new(1 + 0)) + Place::from(Local::new(1)) }; let mut statements = vec![]; @@ -854,7 +887,7 @@ fn build_call_shim<'tcx>( TerminatorKind::Call { func: callee, args, - destination: Place::return_place(), + destination, target: Some(BasicBlock::new(1)), unwind: if let Some(Adjustment::RefMut) = rcvr_adjustment { UnwindAction::Cleanup(BasicBlock::new(3)) @@ -882,7 +915,24 @@ fn build_call_shim<'tcx>( ); } // BB #1/#2 - return - block(&mut blocks, vec![], TerminatorKind::Return, false); + // NOTE: If this is an RPITIT in dyn, we also want to coerce + // the return type of the function into a `dyn*`. + let stmts = if rpitit_shim.is_some() { + vec![Statement { + source_info, + kind: StatementKind::Assign(Box::new(( + Place::return_place(), + Rvalue::Cast( + CastKind::PointerCoercion(PointerCoercion::DynStar, CoercionSource::Implicit), + Operand::Move(destination), + sig.output(), + ), + ))), + }] + } else { + vec![] + }; + block(&mut blocks, stmts, TerminatorKind::Return, false); if let Some(Adjustment::RefMut) = rcvr_adjustment { // BB #3 - drop if closure panics block( @@ -914,7 +964,7 @@ fn build_call_shim<'tcx>( pub(super) fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> Body<'_> { debug_assert!(tcx.is_constructor(ctor_id)); - let param_env = tcx.param_env_reveal_all_normalized(ctor_id); + let typing_env = ty::TypingEnv::post_analysis(tcx, ctor_id); // Normalize the sig. let sig = tcx @@ -922,7 +972,7 @@ pub(super) fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> Body<'_> { .instantiate_identity() .no_bound_vars() .expect("LBR in ADT constructor signature"); - let sig = tcx.normalize_erasing_regions(param_env, sig); + let sig = tcx.normalize_erasing_regions(typing_env, sig); let ty::Adt(adt_def, args) = sig.output().kind() else { bug!("unexpected type for ADT ctor {:?}", sig.output()); diff --git a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs index f1672272862..f01bab75c4a 100644 --- a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs +++ b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs @@ -48,7 +48,7 @@ struct AsyncDestructorCtorShimBuilder<'tcx> { self_ty: Option<Ty<'tcx>>, span: Span, source_info: SourceInfo, - param_env: ty::ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, stack: Vec<Operand<'tcx>>, last_bb: BasicBlock, @@ -86,17 +86,17 @@ impl<'tcx> AsyncDestructorCtorShimBuilder<'tcx> { // Usual case: noop() + unwind resume + return let mut bbs = IndexVec::with_capacity(3); - let param_env = tcx.param_env_reveal_all_normalized(def_id); + let typing_env = ty::TypingEnv::post_analysis(tcx, def_id); AsyncDestructorCtorShimBuilder { tcx, def_id, self_ty, span, source_info, - param_env, + typing_env, stack: Vec::with_capacity(Self::MAX_STACK_LEN), - last_bb: bbs.push(BasicBlockData::new(None)), + last_bb: bbs.push(BasicBlockData::new(None, false)), top_cleanup_bb: match tcx.sess.panic_strategy() { PanicStrategy::Unwind => { // Don't drop input arg because it's just a pointer @@ -422,7 +422,7 @@ impl<'tcx> AsyncDestructorCtorShimBuilder<'tcx> { statements: Vec::new(), terminator: Some(Terminator { source_info, - kind: if self.locals[local].ty.needs_drop(self.tcx, self.param_env) { + kind: if self.locals[local].ty.needs_drop(self.tcx, self.typing_env) { TerminatorKind::Drop { place: local.into(), target: *top_cleanup_bb, diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index 7ed43547e11..4f312ed2aaa 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -523,7 +523,8 @@ impl<'tcx> Visitor<'tcx> for UsedLocals { } StatementKind::SetDiscriminant { ref place, variant_index: _ } - | StatementKind::Deinit(ref place) => { + | StatementKind::Deinit(ref place) + | StatementKind::BackwardIncompatibleDropHint { ref place, reason: _ } => { self.visit_lhs(place, location); } } @@ -560,6 +561,7 @@ fn remove_unused_definitions_helper(used_locals: &mut UsedLocals, body: &mut Bod StatementKind::Assign(box (place, _)) => used_locals.is_used(place.local), StatementKind::SetDiscriminant { ref place, .. } + | StatementKind::BackwardIncompatibleDropHint { ref place, reason: _ } | StatementKind::Deinit(ref place) => used_locals.is_used(place.local), StatementKind::Nop => false, _ => true, @@ -587,6 +589,20 @@ impl<'tcx> MutVisitor<'tcx> for LocalUpdater<'tcx> { self.tcx } + fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) { + if let StatementKind::BackwardIncompatibleDropHint { place, reason: _ } = + &mut statement.kind + { + self.visit_local( + &mut place.local, + PlaceContext::MutatingUse(MutatingUseContext::Store), + location, + ); + } else { + self.super_statement(statement, location); + } + } + fn visit_local(&mut self, l: &mut Local, _: PlaceContext, _: Location) { *l = self.map[*l].unwrap(); } diff --git a/compiler/rustc_mir_transform/src/simplify_branches.rs b/compiler/rustc_mir_transform/src/simplify_branches.rs index e83b4727c48..bea3d0d8557 100644 --- a/compiler/rustc_mir_transform/src/simplify_branches.rs +++ b/compiler/rustc_mir_transform/src/simplify_branches.rs @@ -18,14 +18,14 @@ impl<'tcx> crate::MirPass<'tcx> for SimplifyConstCondition { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { trace!("Running SimplifyConstCondition on {:?}", body.source); - let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); + let typing_env = body.typing_env(tcx); 'blocks: for block in body.basic_blocks_mut() { for stmt in block.statements.iter_mut() { // Simplify `assume` of a known value: either a NOP or unreachable. if let StatementKind::Intrinsic(box ref intrinsic) = stmt.kind && let NonDivergingIntrinsic::Assume(discr) = intrinsic && let Operand::Constant(ref c) = discr - && let Some(constant) = c.const_.try_eval_bool(tcx, param_env) + && let Some(constant) = c.const_.try_eval_bool(tcx, typing_env) { if constant { stmt.make_nop(); @@ -42,7 +42,7 @@ impl<'tcx> crate::MirPass<'tcx> for SimplifyConstCondition { TerminatorKind::SwitchInt { discr: Operand::Constant(ref c), ref targets, .. } => { - let constant = c.const_.try_eval_bits(tcx, param_env); + let constant = c.const_.try_eval_bits(tcx, typing_env); if let Some(constant) = constant { let target = targets.target_for_value(constant); TerminatorKind::Goto { target } @@ -52,7 +52,7 @@ impl<'tcx> crate::MirPass<'tcx> for SimplifyConstCondition { } TerminatorKind::Assert { target, cond: Operand::Constant(ref c), expected, .. - } => match c.const_.try_eval_bool(tcx, param_env) { + } => match c.const_.try_eval_bool(tcx, typing_env) { Some(v) if v == expected => TerminatorKind::Goto { target }, _ => continue, }, diff --git a/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs b/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs index 26496b7f3fe..b6d80173086 100644 --- a/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs +++ b/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs @@ -37,7 +37,7 @@ impl<'tcx> crate::MirPass<'tcx> for SimplifyComparisonIntegral { let opts = helper.find_optimizations(); let mut storage_deads_to_insert = vec![]; let mut storage_deads_to_remove: Vec<(usize, BasicBlock)> = vec![]; - let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); + let typing_env = body.typing_env(tcx); for opt in opts { trace!("SUCCESS: Applying {:?}", opt); // replace terminator with a switchInt that switches on the integer directly @@ -46,7 +46,7 @@ impl<'tcx> crate::MirPass<'tcx> for SimplifyComparisonIntegral { let new_value = match opt.branch_value_scalar { Scalar::Int(int) => { let layout = tcx - .layout_of(param_env.and(opt.branch_value_ty)) + .layout_of(typing_env.as_query_input(opt.branch_value_ty)) .expect("if we have an evaluated constant we must know the layout"); int.to_bits(layout.size) } diff --git a/compiler/rustc_mir_transform/src/sroa.rs b/compiler/rustc_mir_transform/src/sroa.rs index 53bbb122096..52b9ec1e0a3 100644 --- a/compiler/rustc_mir_transform/src/sroa.rs +++ b/compiler/rustc_mir_transform/src/sroa.rs @@ -28,12 +28,12 @@ impl<'tcx> crate::MirPass<'tcx> for ScalarReplacementOfAggregates { } let mut excluded = excluded_locals(body); - let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); + let typing_env = body.typing_env(tcx); loop { debug!(?excluded); - let escaping = escaping_locals(tcx, param_env, &excluded, body); + let escaping = escaping_locals(tcx, typing_env, &excluded, body); debug!(?escaping); - let replacements = compute_flattening(tcx, param_env, body, escaping); + let replacements = compute_flattening(tcx, typing_env, body, escaping); debug!(?replacements); let all_dead_locals = replace_flattened_locals(tcx, body, replacements); if !all_dead_locals.is_empty() { @@ -59,7 +59,7 @@ impl<'tcx> crate::MirPass<'tcx> for ScalarReplacementOfAggregates { /// client code. fn escaping_locals<'tcx>( tcx: TyCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, excluded: &BitSet<Local>, body: &Body<'tcx>, ) -> BitSet<Local> { @@ -84,7 +84,7 @@ fn escaping_locals<'tcx>( // niche, so we do not want to automatically exclude it. return false; } - let Ok(layout) = tcx.layout_of(param_env.and(ty)) else { + let Ok(layout) = tcx.layout_of(typing_env.as_query_input(ty)) else { // We can't get the layout return true; }; @@ -196,7 +196,7 @@ impl<'tcx> ReplacementMap<'tcx> { /// The replacement will be done later in `ReplacementVisitor`. fn compute_flattening<'tcx>( tcx: TyCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, body: &mut Body<'tcx>, escaping: BitSet<Local>, ) -> ReplacementMap<'tcx> { @@ -208,7 +208,7 @@ fn compute_flattening<'tcx>( } let decl = body.local_decls[local].clone(); let ty = decl.ty; - iter_fields(ty, tcx, param_env, |variant, field, field_ty| { + iter_fields(ty, tcx, typing_env, |variant, field, field_ty| { if variant.is_some() { // Downcasts are currently not supported. return; diff --git a/compiler/rustc_mir_transform/src/ssa.rs b/compiler/rustc_mir_transform/src/ssa.rs index 84df666e34a..5653aef0aae 100644 --- a/compiler/rustc_mir_transform/src/ssa.rs +++ b/compiler/rustc_mir_transform/src/ssa.rs @@ -13,7 +13,7 @@ use rustc_middle::bug; use rustc_middle::middle::resolve_bound_vars::Set1; use rustc_middle::mir::visit::*; use rustc_middle::mir::*; -use rustc_middle::ty::{ParamEnv, TyCtxt}; +use rustc_middle::ty::{self, TyCtxt}; use tracing::{debug, instrument, trace}; pub(super) struct SsaLocals { @@ -42,7 +42,7 @@ impl SsaLocals { pub(super) fn new<'tcx>( tcx: TyCtxt<'tcx>, body: &Body<'tcx>, - param_env: ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, ) -> SsaLocals { let assignment_order = Vec::with_capacity(body.local_decls.len()); @@ -80,7 +80,7 @@ impl SsaLocals { // have already been marked as non-SSA. debug!(?visitor.borrowed_locals); for local in visitor.borrowed_locals.iter() { - if !body.local_decls[local].ty.is_freeze(tcx, param_env) { + if !body.local_decls[local].ty.is_freeze(tcx, typing_env) { visitor.assignments[local] = Set1::Many; } } diff --git a/compiler/rustc_mir_transform/src/strip_debuginfo.rs b/compiler/rustc_mir_transform/src/strip_debuginfo.rs new file mode 100644 index 00000000000..438c75726bb --- /dev/null +++ b/compiler/rustc_mir_transform/src/strip_debuginfo.rs @@ -0,0 +1,34 @@ +use rustc_middle::mir::*; +use rustc_middle::ty::TyCtxt; +use rustc_session::config::MirStripDebugInfo; + +/// Conditionally remove some of the VarDebugInfo in MIR. +/// +/// In particular, stripping non-parameter debug info for tiny, primitive-like +/// methods in core saves work later, and nobody ever wanted to use it anyway. +pub(super) struct StripDebugInfo; + +impl<'tcx> crate::MirPass<'tcx> for StripDebugInfo { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.opts.unstable_opts.mir_strip_debuginfo != MirStripDebugInfo::None + } + + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + match tcx.sess.opts.unstable_opts.mir_strip_debuginfo { + MirStripDebugInfo::None => return, + MirStripDebugInfo::AllLocals => {} + MirStripDebugInfo::LocalsInTinyFunctions + if let TerminatorKind::Return { .. } = + body.basic_blocks[START_BLOCK].terminator().kind => {} + MirStripDebugInfo::LocalsInTinyFunctions => return, + } + + body.var_debug_info.retain(|vdi| { + matches!( + vdi.value, + VarDebugInfoContents::Place(place) + if place.local.as_usize() <= body.arg_count && place.local != RETURN_PLACE, + ) + }); + } +} diff --git a/compiler/rustc_mir_transform/src/unreachable_enum_branching.rs b/compiler/rustc_mir_transform/src/unreachable_enum_branching.rs index 3011af4d9d7..55dcad0680a 100644 --- a/compiler/rustc_mir_transform/src/unreachable_enum_branching.rs +++ b/compiler/rustc_mir_transform/src/unreachable_enum_branching.rs @@ -54,6 +54,10 @@ fn variant_discriminants<'tcx>( tcx: TyCtxt<'tcx>, ) -> FxHashSet<u128> { match &layout.variants { + Variants::Empty => { + // Uninhabited, no valid discriminant. + FxHashSet::default() + } Variants::Single { index } => { let mut res = FxHashSet::default(); res.insert( @@ -92,9 +96,7 @@ impl<'tcx> crate::MirPass<'tcx> for UnreachableEnumBranching { let Some(discriminant_ty) = get_switched_on_type(bb_data, tcx, body) else { continue }; - let layout = tcx.layout_of( - tcx.param_env_reveal_all_normalized(body.source.def_id()).and(discriminant_ty), - ); + let layout = tcx.layout_of(body.typing_env(tcx).as_query_input(discriminant_ty)); let mut allowed_variants = if let Ok(layout) = layout { // Find allowed variants based on uninhabited. diff --git a/compiler/rustc_mir_transform/src/unreachable_prop.rs b/compiler/rustc_mir_transform/src/unreachable_prop.rs index 9cd32459c7b..734703ec78b 100644 --- a/compiler/rustc_mir_transform/src/unreachable_prop.rs +++ b/compiler/rustc_mir_transform/src/unreachable_prop.rs @@ -43,12 +43,6 @@ impl crate::MirPass<'_> for UnreachablePropagation { } } - if !tcx - .consider_optimizing(|| format!("UnreachablePropagation {:?} ", body.source.def_id())) - { - return; - } - patch.apply(body); // We do want do keep some unreachable blocks, but make them empty. diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index ae4e6ea6a74..bce015046e1 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -12,8 +12,7 @@ use rustc_middle::mir::visit::{NonUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::{ - self, CoroutineArgsExt, InstanceKind, ParamEnv, ScalarInt, Ty, TyCtxt, TypeVisitableExt, - Variance, + self, CoroutineArgsExt, InstanceKind, ScalarInt, Ty, TyCtxt, TypeVisitableExt, Variance, }; use rustc_middle::{bug, span_bug}; use rustc_trait_selection::traits::ObligationCtxt; @@ -30,12 +29,6 @@ enum EdgeKind { pub(super) struct Validator { /// Describes at which point in the pipeline this validation is happening. pub when: String, - /// The phase for which we are upholding the dialect. If the given phase forbids a specific - /// element, this validator will now emit errors if that specific element is encountered. - /// Note that phases that change the dialect cause all *following* phases to check the - /// invariants of the new dialect. A phase that changes dialects never checks the new invariants - /// itself. - pub mir_phase: MirPhase, } impl<'tcx> crate::MirPass<'tcx> for Validator { @@ -48,9 +41,8 @@ impl<'tcx> crate::MirPass<'tcx> for Validator { return; } let def_id = body.source.def_id(); - let mir_phase = self.mir_phase; - let param_env = mir_phase.param_env(tcx, def_id); - let can_unwind = if mir_phase <= MirPhase::Runtime(RuntimePhase::Initial) { + let typing_env = body.typing_env(tcx); + let can_unwind = if body.phase <= MirPhase::Runtime(RuntimePhase::Initial) { // In this case `AbortUnwindingCalls` haven't yet been executed. true } else if !tcx.def_kind(def_id).is_fn_like() { @@ -64,9 +56,7 @@ impl<'tcx> crate::MirPass<'tcx> for Validator { ty::Coroutine(..) => ExternAbi::Rust, // No need to do MIR validation on error bodies ty::Error(_) => return, - _ => { - span_bug!(body.span, "unexpected body ty: {:?} phase {:?}", body_ty, mir_phase) - } + _ => span_bug!(body.span, "unexpected body ty: {body_ty:?}"), }; ty::layout::fn_can_unwind(tcx, Some(def_id), body_abi) @@ -76,7 +66,6 @@ impl<'tcx> crate::MirPass<'tcx> for Validator { when: &self.when, body, tcx, - mir_phase, unwind_edge_count: 0, reachable_blocks: traversal::reachable_as_bitset(body), value_cache: FxHashSet::default(), @@ -86,7 +75,7 @@ impl<'tcx> crate::MirPass<'tcx> for Validator { cfg_checker.check_cleanup_control_flow(); // Also run the TypeChecker. - for (location, msg) in validate_types(tcx, self.mir_phase, param_env, body, body) { + for (location, msg) in validate_types(tcx, typing_env, body, body) { cfg_checker.fail(location, msg); } @@ -107,7 +96,6 @@ struct CfgChecker<'a, 'tcx> { when: &'a str, body: &'a Body<'tcx>, tcx: TyCtxt<'tcx>, - mir_phase: MirPhase, unwind_edge_count: usize, reachable_blocks: BitSet<BasicBlock>, value_cache: FxHashSet<u128>, @@ -294,7 +282,7 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { match &statement.kind { StatementKind::AscribeUserType(..) => { - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { + if self.body.phase >= MirPhase::Runtime(RuntimePhase::Initial) { self.fail( location, "`AscribeUserType` should have been removed after drop lowering phase", @@ -302,7 +290,7 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { } } StatementKind::FakeRead(..) => { - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { + if self.body.phase >= MirPhase::Runtime(RuntimePhase::Initial) { self.fail( location, "`FakeRead` should have been removed after drop lowering phase", @@ -310,17 +298,17 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { } } StatementKind::SetDiscriminant { .. } => { - if self.mir_phase < MirPhase::Runtime(RuntimePhase::Initial) { + if self.body.phase < MirPhase::Runtime(RuntimePhase::Initial) { self.fail(location, "`SetDiscriminant`is not allowed until deaggregation"); } } StatementKind::Deinit(..) => { - if self.mir_phase < MirPhase::Runtime(RuntimePhase::Initial) { + if self.body.phase < MirPhase::Runtime(RuntimePhase::Initial) { self.fail(location, "`Deinit`is not allowed until deaggregation"); } } StatementKind::Retag(kind, _) => { - // FIXME(JakobDegen) The validator should check that `self.mir_phase < + // FIXME(JakobDegen) The validator should check that `self.body.phase < // DropsLowered`. However, this causes ICEs with generation of drop shims, which // seem to fail to set their `MirPhase` correctly. if matches!(kind, RetagKind::TwoPhase) { @@ -328,7 +316,7 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { } } StatementKind::Coverage(kind) => { - if self.mir_phase >= MirPhase::Analysis(AnalysisPhase::PostCleanup) + if self.body.phase >= MirPhase::Analysis(AnalysisPhase::PostCleanup) && let CoverageKind::BlockMarker { .. } | CoverageKind::SpanMarker { .. } = kind { self.fail( @@ -343,6 +331,7 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { | StatementKind::Intrinsic(_) | StatementKind::ConstEvalCounter | StatementKind::PlaceMention(..) + | StatementKind::BackwardIncompatibleDropHint { .. } | StatementKind::Nop => {} } @@ -390,7 +379,7 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { // the return edge from the call. FIXME(tmiasko): Since this is a strictly code // generation concern, the code generation should be responsible for handling // it. - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Optimized) + if self.body.phase >= MirPhase::Runtime(RuntimePhase::Optimized) && self.is_critical_call_edge(target, unwind) { self.fail( @@ -439,7 +428,7 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { if self.body.coroutine.is_none() { self.fail(location, "`Yield` cannot appear outside coroutine bodies"); } - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { + if self.body.phase >= MirPhase::Runtime(RuntimePhase::Initial) { self.fail(location, "`Yield` should have been replaced by coroutine lowering"); } self.check_edge(location, *resume, EdgeKind::Normal); @@ -448,7 +437,7 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { } } TerminatorKind::FalseEdge { real_target, imaginary_target } => { - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { + if self.body.phase >= MirPhase::Runtime(RuntimePhase::Initial) { self.fail( location, "`FalseEdge` should have been removed after drop elaboration", @@ -458,7 +447,7 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { self.check_edge(location, *imaginary_target, EdgeKind::Normal); } TerminatorKind::FalseUnwind { real_target, unwind } => { - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { + if self.body.phase >= MirPhase::Runtime(RuntimePhase::Initial) { self.fail( location, "`FalseUnwind` should have been removed after drop elaboration", @@ -477,7 +466,7 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { if self.body.coroutine.is_none() { self.fail(location, "`CoroutineDrop` cannot appear outside coroutine bodies"); } - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { + if self.body.phase >= MirPhase::Runtime(RuntimePhase::Initial) { self.fail( location, "`CoroutineDrop` should have been replaced by coroutine lowering", @@ -531,13 +520,11 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { /// `optimized_mir` is available. pub(super) fn validate_types<'tcx>( tcx: TyCtxt<'tcx>, - mir_phase: MirPhase, - param_env: ty::ParamEnv<'tcx>, + typing_env: ty::TypingEnv<'tcx>, body: &Body<'tcx>, caller_body: &Body<'tcx>, ) -> Vec<(Location, String)> { - let mut type_checker = - TypeChecker { body, caller_body, tcx, param_env, mir_phase, failures: Vec::new() }; + let mut type_checker = TypeChecker { body, caller_body, tcx, typing_env, failures: Vec::new() }; type_checker.visit_body(body); type_checker.failures } @@ -546,8 +533,7 @@ struct TypeChecker<'a, 'tcx> { body: &'a Body<'tcx>, caller_body: &'a Body<'tcx>, tcx: TyCtxt<'tcx>, - param_env: ParamEnv<'tcx>, - mir_phase: MirPhase, + typing_env: ty::TypingEnv<'tcx>, failures: Vec<(Location, String)>, } @@ -576,20 +562,13 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { // After borrowck subtyping should be fully explicit via // `Subtype` projections. - let variance = if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { + let variance = if self.body.phase >= MirPhase::Runtime(RuntimePhase::Initial) { Variance::Invariant } else { Variance::Covariant }; - crate::util::relate_types( - self.tcx, - self.body.typing_mode(self.tcx), - self.param_env, - variance, - src, - dest, - ) + crate::util::relate_types(self.tcx, self.typing_env, variance, src, dest) } /// Check that the given predicate definitely holds in the param-env of this MIR body. @@ -608,12 +587,12 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { return true; } - let infcx = self.tcx.infer_ctxt().build(self.body.typing_mode(self.tcx)); + let (infcx, param_env) = self.tcx.infer_ctxt().build_with_typing_env(self.typing_env); let ocx = ObligationCtxt::new(&infcx); ocx.register_obligation(Obligation::new( self.tcx, ObligationCause::dummy(), - self.param_env, + param_env, pred, )); ocx.select_all_or_error().is_empty() @@ -624,13 +603,13 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { fn visit_operand(&mut self, operand: &Operand<'tcx>, location: Location) { // This check is somewhat expensive, so only run it when -Zvalidate-mir is passed. if self.tcx.sess.opts.unstable_opts.validate_mir - && self.mir_phase < MirPhase::Runtime(RuntimePhase::Initial) + && self.body.phase < MirPhase::Runtime(RuntimePhase::Initial) { // `Operand::Copy` is only supposed to be used with `Copy` types. if let Operand::Copy(place) = operand { let ty = place.ty(&self.body.local_decls, self.tcx).ty; - if !ty.is_copy_modulo_regions(self.tcx, self.param_env) { + if !self.tcx.type_is_copy_modulo_regions(self.typing_env, ty) { self.fail(location, format!("`Operand::Copy` with non-`Copy` type {ty}")); } } @@ -648,11 +627,11 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { ) { match elem { ProjectionElem::OpaqueCast(ty) - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) => + if self.body.phase >= MirPhase::Runtime(RuntimePhase::Initial) => { self.fail( location, - format!("explicit opaque type cast to `{ty}` after `RevealAll`"), + format!("explicit opaque type cast to `{ty}` after `PostAnalysisNormalize`"), ) } ProjectionElem::Index(index) => { @@ -662,7 +641,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } ProjectionElem::Deref - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::PostCleanup) => + if self.body.phase >= MirPhase::Runtime(RuntimePhase::PostCleanup) => { let base_ty = place_ref.ty(&self.body.local_decls, self.tcx).ty; @@ -802,8 +781,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { ProjectionElem::Subtype(ty) => { if !util::sub_types( self.tcx, - self.body.typing_mode(self.tcx), - self.param_env, + self.typing_env, ty, place_ref.ty(&self.body.local_decls, self.tcx).ty, ) { @@ -863,7 +841,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { // Set off any `bug!`s in the type computation code let _ = place.ty(&self.body.local_decls, self.tcx); - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) + if self.body.phase >= MirPhase::Runtime(RuntimePhase::Initial) && place.projection.len() > 1 && cntxt != PlaceContext::NonUse(NonUseContext::VarDebugInfo) && place.projection[1..].contains(&ProjectionElem::Deref) @@ -916,7 +894,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { assert!(adt_def.is_union()); assert_eq!(idx, FIRST_VARIANT); let dest_ty = self.tcx.normalize_erasing_regions( - self.param_env, + self.typing_env, adt_def.non_enum_variant().fields[field].ty(self.tcx, args), ); if let [field] = fields.raw.as_slice() { @@ -938,7 +916,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { for (src, dest) in std::iter::zip(fields, &variant.fields) { let dest_ty = self .tcx - .normalize_erasing_regions(self.param_env, dest.ty(self.tcx, args)); + .normalize_erasing_regions(self.typing_env, dest.ty(self.tcx, args)); if !self.mir_assign_valid_types(src.ty(self.body, self.tcx), dest_ty) { self.fail(location, "adt field has the wrong type"); } @@ -981,7 +959,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } AggregateKind::RawPtr(pointee_ty, mutability) => { - if !matches!(self.mir_phase, MirPhase::Runtime(_)) { + if !matches!(self.body.phase, MirPhase::Runtime(_)) { // It would probably be fine to support this in earlier phases, but at the // time of writing it's only ever introduced from intrinsic lowering, so // earlier things just `bug!` on it. @@ -997,7 +975,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } // FIXME: check `Thin` instead of `Sized` - if !in_pointee.is_sized(self.tcx, self.param_env) { + if !in_pointee.is_sized(self.tcx, self.typing_env) { self.fail(location, "input pointer must be thin"); } } else { @@ -1012,7 +990,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { if !self.mir_assign_valid_types(metadata_ty, self.tcx.types.usize) { self.fail(location, "slice metadata must be usize"); } - } else if pointee_ty.is_sized(self.tcx, self.param_env) { + } else if pointee_ty.is_sized(self.tcx, self.typing_env) { if metadata_ty != self.tcx.types.unit { self.fail(location, "metadata for pointer-to-thin must be unit"); } @@ -1023,7 +1001,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } }, Rvalue::Ref(_, BorrowKind::Fake(_), _) => { - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { + if self.body.phase >= MirPhase::Runtime(RuntimePhase::Initial) { self.fail( location, "`Assign` statement with a `Fake` borrow should have been removed in runtime MIR", @@ -1137,14 +1115,6 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { ); } UnOp::PtrMetadata => { - if !matches!(self.mir_phase, MirPhase::Runtime(_)) { - // It would probably be fine to support this in earlier phases, but at - // the time of writing it's only ever introduced from intrinsic - // lowering or other runtime-phase optimization passes, so earlier - // things can just `bug!` on it. - self.fail(location, "PtrMetadata should be in runtime MIR only"); - } - check_kinds!( a, "Cannot PtrMetadata non-pointer non-reference type {:?}", @@ -1213,7 +1183,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { "CastKind::{kind:?} output must be a raw const pointer, not {:?}", ty::RawPtr(_, Mutability::Not) ); - if self.mir_phase >= MirPhase::Analysis(AnalysisPhase::PostCleanup) { + if self.body.phase >= MirPhase::Analysis(AnalysisPhase::PostCleanup) { self.fail(location, format!("After borrowck, MIR disallows {kind:?}")); } } @@ -1229,7 +1199,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { "CastKind::{kind:?} output must be a raw pointer, not {:?}", ty::RawPtr(..) ); - if self.mir_phase >= MirPhase::Analysis(AnalysisPhase::PostCleanup) { + if self.body.phase >= MirPhase::Analysis(AnalysisPhase::PostCleanup) { self.fail(location, format!("After borrowck, MIR disallows {kind:?}")); } } @@ -1295,14 +1265,14 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } CastKind::Transmute => { - if let MirPhase::Runtime(..) = self.mir_phase { + if let MirPhase::Runtime(..) = self.body.phase { // Unlike `mem::transmute`, a MIR `Transmute` is well-formed // for any two `Sized` types, just potentially UB to run. if !self .tcx - .normalize_erasing_regions(self.param_env, op_ty) - .is_sized(self.tcx, self.param_env) + .normalize_erasing_regions(self.typing_env, op_ty) + .is_sized(self.tcx, self.typing_env) { self.fail( location, @@ -1311,8 +1281,8 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } if !self .tcx - .normalize_erasing_regions(self.param_env, *target_type) - .is_sized(self.tcx, self.param_env) + .normalize_erasing_regions(self.typing_env, *target_type) + .is_sized(self.tcx, self.typing_env) { self.fail( location, @@ -1324,7 +1294,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { location, format!( "Transmute is not supported in non-runtime phase {:?}.", - self.mir_phase + self.body.phase ), ); } @@ -1353,7 +1323,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { return; }; - current_ty = self.tcx.normalize_erasing_regions(self.param_env, f_ty); + current_ty = self.tcx.normalize_erasing_regions(self.typing_env, f_ty); } ty::Adt(adt_def, args) => { let Some(field) = adt_def.variant(variant).fields.get(field) else { @@ -1362,7 +1332,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { }; let f_ty = field.ty(self.tcx, args); - current_ty = self.tcx.normalize_erasing_regions(self.param_env, f_ty); + current_ty = self.tcx.normalize_erasing_regions(self.typing_env, f_ty); } _ => { self.fail( @@ -1411,7 +1381,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } StatementKind::AscribeUserType(..) => { - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { + if self.body.phase >= MirPhase::Runtime(RuntimePhase::Initial) { self.fail( location, "`AscribeUserType` should have been removed after drop lowering phase", @@ -1419,7 +1389,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } StatementKind::FakeRead(..) => { - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) { + if self.body.phase >= MirPhase::Runtime(RuntimePhase::Initial) { self.fail( location, "`FakeRead` should have been removed after drop lowering phase", @@ -1470,7 +1440,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } StatementKind::SetDiscriminant { place, .. } => { - if self.mir_phase < MirPhase::Runtime(RuntimePhase::Initial) { + if self.body.phase < MirPhase::Runtime(RuntimePhase::Initial) { self.fail(location, "`SetDiscriminant`is not allowed until deaggregation"); } let pty = place.ty(&self.body.local_decls, self.tcx).ty.kind(); @@ -1484,12 +1454,12 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } StatementKind::Deinit(..) => { - if self.mir_phase < MirPhase::Runtime(RuntimePhase::Initial) { + if self.body.phase < MirPhase::Runtime(RuntimePhase::Initial) { self.fail(location, "`Deinit`is not allowed until deaggregation"); } } StatementKind::Retag(kind, _) => { - // FIXME(JakobDegen) The validator should check that `self.mir_phase < + // FIXME(JakobDegen) The validator should check that `self.body.phase < // DropsLowered`. However, this causes ICEs with generation of drop shims, which // seem to fail to set their `MirPhase` correctly. if matches!(kind, RetagKind::TwoPhase) { @@ -1501,6 +1471,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { | StatementKind::Coverage(_) | StatementKind::ConstEvalCounter | StatementKind::PlaceMention(..) + | StatementKind::BackwardIncompatibleDropHint { .. } | StatementKind::Nop => {} } |