diff options
Diffstat (limited to 'compiler/rustc_mir_transform')
19 files changed, 244 insertions, 150 deletions
diff --git a/compiler/rustc_mir_transform/src/add_subtyping_projections.rs b/compiler/rustc_mir_transform/src/add_subtyping_projections.rs new file mode 100644 index 00000000000..1cc049d5a22 --- /dev/null +++ b/compiler/rustc_mir_transform/src/add_subtyping_projections.rs @@ -0,0 +1,66 @@ +use crate::MirPass; +use rustc_index::IndexVec; +use rustc_middle::mir::patch::MirPatch; +use rustc_middle::mir::visit::MutVisitor; +use rustc_middle::mir::*; +use rustc_middle::ty::TyCtxt; + +pub struct Subtyper; + +pub struct SubTypeChecker<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + patcher: MirPatch<'tcx>, + local_decls: &'a IndexVec<Local, LocalDecl<'tcx>>, +} + +impl<'a, 'tcx> MutVisitor<'tcx> for SubTypeChecker<'a, 'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_assign( + &mut self, + place: &mut Place<'tcx>, + rvalue: &mut Rvalue<'tcx>, + location: Location, + ) { + let mut place_ty = place.ty(self.local_decls, self.tcx).ty; + let mut rval_ty = rvalue.ty(self.local_decls, self.tcx); + // Not erasing this causes `Free Regions` errors in validator, + // when rval is `ReStatic`. + rval_ty = self.tcx.erase_regions_ty(rval_ty); + place_ty = self.tcx.erase_regions(place_ty); + if place_ty != rval_ty { + let temp = self + .patcher + .new_temp(rval_ty, self.local_decls[place.as_ref().local].source_info.span); + let new_place = Place::from(temp); + self.patcher.add_assign(location, new_place, rvalue.clone()); + let subtyped = new_place.project_deeper(&[ProjectionElem::Subtype(place_ty)], self.tcx); + *rvalue = Rvalue::Use(Operand::Move(subtyped)); + } + } +} + +// Aim here is to do this kind of transformation: +// +// let place: place_ty = rval; +// // gets transformed to +// let temp: rval_ty = rval; +// let place: place_ty = temp as place_ty; +// +pub fn subtype_finder<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let patch = MirPatch::new(body); + let mut checker = SubTypeChecker { tcx, patcher: patch, local_decls: &body.local_decls }; + + for (bb, data) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() { + checker.visit_basic_block_data(bb, data); + } + checker.patcher.apply(body); +} + +impl<'tcx> MirPass<'tcx> for Subtyper { + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + subtype_finder(tcx, body); + } +} diff --git a/compiler/rustc_mir_transform/src/check_unsafety.rs b/compiler/rustc_mir_transform/src/check_unsafety.rs index bacabc62ee4..c428007707e 100644 --- a/compiler/rustc_mir_transform/src/check_unsafety.rs +++ b/compiler/rustc_mir_transform/src/check_unsafety.rs @@ -179,7 +179,7 @@ impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> { // Check the base local: it might be an unsafe-to-access static. We only check derefs of the // temporary holding the static pointer to avoid duplicate errors // <https://github.com/rust-lang/rust/pull/78068#issuecomment-731753506>. - if decl.internal && place.projection.first() == Some(&ProjectionElem::Deref) { + if place.projection.first() == Some(&ProjectionElem::Deref) { // If the projection root is an artificial local that we introduced when // desugaring `static`, give a more specific error message // (avoid the general "raw pointer" clause below, that would only be confusing). diff --git a/compiler/rustc_mir_transform/src/coverage/counters.rs b/compiler/rustc_mir_transform/src/coverage/counters.rs index d56d4ad4f1e..78845af0162 100644 --- a/compiler/rustc_mir_transform/src/coverage/counters.rs +++ b/compiler/rustc_mir_transform/src/coverage/counters.rs @@ -1,10 +1,8 @@ use super::Error; use super::graph; -use super::spans; use graph::{BasicCoverageBlock, BcbBranch, CoverageGraph, TraverseCoverageGraphWithLoops}; -use spans::CoverageSpan; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::graph::WithNumNodes; @@ -93,14 +91,14 @@ impl CoverageCounters { } /// Makes [`BcbCounter`] `Counter`s and `Expressions` for the `BasicCoverageBlock`s directly or - /// indirectly associated with `CoverageSpans`, and accumulates additional `Expression`s + /// indirectly associated with coverage spans, and accumulates additional `Expression`s /// representing intermediate values. pub fn make_bcb_counters( &mut self, basic_coverage_blocks: &CoverageGraph, - coverage_spans: &[CoverageSpan], + bcb_has_coverage_spans: impl Fn(BasicCoverageBlock) -> bool, ) -> Result<(), Error> { - MakeBcbCounters::new(self, basic_coverage_blocks).make_bcb_counters(coverage_spans) + MakeBcbCounters::new(self, basic_coverage_blocks).make_bcb_counters(bcb_has_coverage_spans) } fn make_counter(&mut self) -> BcbCounter { @@ -113,14 +111,10 @@ impl CoverageCounters { BcbCounter::Expression { id, lhs, op, rhs } } - pub fn make_identity_counter(&mut self, counter_operand: Operand) -> BcbCounter { - self.make_expression(counter_operand, Op::Add, Operand::Zero) - } - /// Counter IDs start from one and go up. fn next_counter(&mut self) -> CounterId { let next = self.next_counter_id; - self.next_counter_id = next.next_id(); + self.next_counter_id = self.next_counter_id + 1; next } @@ -128,7 +122,7 @@ impl CoverageCounters { /// (Counter IDs and Expression IDs are distinguished by the `Operand` enum.) fn next_expression(&mut self) -> ExpressionId { let next = self.next_expression_id; - self.next_expression_id = next.next_id(); + self.next_expression_id = self.next_expression_id + 1; next } @@ -208,7 +202,7 @@ impl CoverageCounters { } /// Traverse the `CoverageGraph` and add either a `Counter` or `Expression` to every BCB, to be -/// injected with `CoverageSpan`s. `Expressions` have no runtime overhead, so if a viable expression +/// injected with coverage spans. `Expressions` have no runtime overhead, so if a viable expression /// (adding or subtracting two other counters or expressions) can compute the same result as an /// embedded counter, an `Expression` should be used. struct MakeBcbCounters<'a> { @@ -234,17 +228,14 @@ impl<'a> MakeBcbCounters<'a> { /// Returns any non-code-span expressions created to represent intermediate values (such as to /// add two counters so the result can be subtracted from another counter), or an Error with /// message for subsequent debugging. - fn make_bcb_counters(&mut self, coverage_spans: &[CoverageSpan]) -> Result<(), Error> { + fn make_bcb_counters( + &mut self, + bcb_has_coverage_spans: impl Fn(BasicCoverageBlock) -> bool, + ) -> Result<(), Error> { debug!("make_bcb_counters(): adding a counter or expression to each BasicCoverageBlock"); - let num_bcbs = self.basic_coverage_blocks.num_nodes(); - - let mut bcbs_with_coverage = BitSet::new_empty(num_bcbs); - for covspan in coverage_spans { - bcbs_with_coverage.insert(covspan.bcb); - } // Walk the `CoverageGraph`. For each `BasicCoverageBlock` node with an associated - // `CoverageSpan`, add a counter. If the `BasicCoverageBlock` branches, add a counter or + // coverage span, add a counter. If the `BasicCoverageBlock` branches, add a counter or // expression to each branch `BasicCoverageBlock` (if the branch BCB has only one incoming // edge) or edge from the branching BCB to the branch BCB (if the branch BCB has multiple // incoming edges). @@ -255,8 +246,8 @@ impl<'a> MakeBcbCounters<'a> { // the current BCB is in one or more nested loops or not. let mut traversal = TraverseCoverageGraphWithLoops::new(&self.basic_coverage_blocks); while let Some(bcb) = traversal.next(self.basic_coverage_blocks) { - if bcbs_with_coverage.contains(bcb) { - debug!("{:?} has at least one `CoverageSpan`. Get or make its counter", bcb); + if bcb_has_coverage_spans(bcb) { + debug!("{:?} has at least one coverage span. Get or make its counter", bcb); let branching_counter_operand = self.get_or_make_counter_operand(bcb)?; if self.bcb_needs_branch_counters(bcb) { @@ -264,7 +255,7 @@ impl<'a> MakeBcbCounters<'a> { } } else { debug!( - "{:?} does not have any `CoverageSpan`s. A counter will only be added if \ + "{:?} does not have any coverage spans. A counter will only be added if \ and when a covered BCB has an expression dependency.", bcb, ); diff --git a/compiler/rustc_mir_transform/src/coverage/graph.rs b/compiler/rustc_mir_transform/src/coverage/graph.rs index ff2254d6941..812633348e3 100644 --- a/compiler/rustc_mir_transform/src/coverage/graph.rs +++ b/compiler/rustc_mir_transform/src/coverage/graph.rs @@ -288,9 +288,9 @@ rustc_index::newtype_index! { /// not relevant to coverage analysis. `FalseUnwind`, for example, can be treated the same as /// a `Goto`, and merged with its successor into the same BCB. /// -/// Each BCB with at least one computed `CoverageSpan` will have no more than one `Counter`. +/// Each BCB with at least one computed coverage span will have no more than one `Counter`. /// In some cases, a BCB's execution count can be computed by `Expression`. Additional -/// disjoint `CoverageSpan`s in a BCB can also be counted by `Expression` (by adding `ZERO` +/// disjoint coverage spans in a BCB can also be counted by `Expression` (by adding `ZERO` /// to the BCB's primary counter or expression). /// /// The BCB CFG is critical to simplifying the coverage analysis by ensuring graph path-based diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs index c75d33eeb31..abf13519e9e 100644 --- a/compiler/rustc_mir_transform/src/coverage/mod.rs +++ b/compiler/rustc_mir_transform/src/coverage/mod.rs @@ -9,13 +9,11 @@ mod tests; use self::counters::{BcbCounter, CoverageCounters}; use self::graph::{BasicCoverageBlock, BasicCoverageBlockData, CoverageGraph}; -use self::spans::{CoverageSpan, CoverageSpans}; +use self::spans::CoverageSpans; use crate::MirPass; -use rustc_data_structures::graph::WithNumNodes; use rustc_data_structures::sync::Lrc; -use rustc_index::IndexVec; use rustc_middle::hir; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir::coverage::*; @@ -154,7 +152,7 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { let body_span = self.body_span; //////////////////////////////////////////////////// - // Compute `CoverageSpan`s from the `CoverageGraph`. + // Compute coverage spans from the `CoverageGraph`. let coverage_spans = CoverageSpans::generate_coverage_spans( &self.mir_body, fn_sig_span, @@ -164,32 +162,33 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { //////////////////////////////////////////////////// // Create an optimized mix of `Counter`s and `Expression`s for the `CoverageGraph`. Ensure - // every `CoverageSpan` has a `Counter` or `Expression` assigned to its `BasicCoverageBlock` + // every coverage span has a `Counter` or `Expression` assigned to its `BasicCoverageBlock` // and all `Expression` dependencies (operands) are also generated, for any other - // `BasicCoverageBlock`s not already associated with a `CoverageSpan`. + // `BasicCoverageBlock`s not already associated with a coverage span. // // Intermediate expressions (used to compute other `Expression` values), which have no // direct association with any `BasicCoverageBlock`, are accumulated inside `coverage_counters`. + let bcb_has_coverage_spans = |bcb| coverage_spans.bcb_has_coverage_spans(bcb); let result = self .coverage_counters - .make_bcb_counters(&mut self.basic_coverage_blocks, &coverage_spans); + .make_bcb_counters(&mut self.basic_coverage_blocks, bcb_has_coverage_spans); if let Ok(()) = result { //////////////////////////////////////////////////// - // Remove the counter or edge counter from of each `CoverageSpan`s associated + // Remove the counter or edge counter from of each coverage cpan's associated // `BasicCoverageBlock`, and inject a `Coverage` statement into the MIR. // - // `Coverage` statements injected from `CoverageSpan`s will include the code regions + // `Coverage` statements injected from coverage spans will include the code regions // (source code start and end positions) to be counted by the associated counter. // - // These `CoverageSpan`-associated counters are removed from their associated + // These coverage-span-associated counters are removed from their associated // `BasicCoverageBlock`s so that the only remaining counters in the `CoverageGraph` // are indirect counters (to be injected next, without associated code regions). - self.inject_coverage_span_counters(coverage_spans); + self.inject_coverage_span_counters(&coverage_spans); //////////////////////////////////////////////////// // For any remaining `BasicCoverageBlock` counters (that were not associated with - // any `CoverageSpan`), inject `Coverage` statements (_without_ code region `Span`s) + // any coverage span), inject `Coverage` statements (_without_ code region spans) // to ensure `BasicCoverageBlock` counters that other `Expression`s may depend on // are in fact counted, even though they don't directly contribute to counting // their own independent code region's coverage. @@ -214,47 +213,40 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { } } - /// Inject a counter for each `CoverageSpan`. There can be multiple `CoverageSpan`s for a given - /// BCB, but only one actual counter needs to be incremented per BCB. `bb_counters` maps each - /// `bcb` to its `Counter`, when injected. Subsequent `CoverageSpan`s for a BCB that already has - /// a `Counter` will inject an `Expression` instead, and compute its value by adding `ZERO` to - /// the BCB `Counter` value. - fn inject_coverage_span_counters(&mut self, coverage_spans: Vec<CoverageSpan>) { + /// Injects a single [`StatementKind::Coverage`] for each BCB that has one + /// or more coverage spans. + fn inject_coverage_span_counters(&mut self, coverage_spans: &CoverageSpans) { let tcx = self.tcx; let source_map = tcx.sess.source_map(); let body_span = self.body_span; let file_name = Symbol::intern(&self.source_file.name.prefer_remapped().to_string_lossy()); - let mut bcb_counters = IndexVec::from_elem_n(None, self.basic_coverage_blocks.num_nodes()); - for covspan in coverage_spans { - let bcb = covspan.bcb; - let span = covspan.span; - let counter_kind = if let Some(&counter_operand) = bcb_counters[bcb].as_ref() { - self.coverage_counters.make_identity_counter(counter_operand) - } else if let Some(counter_kind) = self.coverage_counters.take_bcb_counter(bcb) { - bcb_counters[bcb] = Some(counter_kind.as_operand()); - counter_kind - } else { + for (bcb, spans) in coverage_spans.bcbs_with_coverage_spans() { + let counter_kind = self.coverage_counters.take_bcb_counter(bcb).unwrap_or_else(|| { bug!("Every BasicCoverageBlock should have a Counter or Expression"); - }; + }); - let code_region = make_code_region(source_map, file_name, span, body_span); + // Convert the coverage spans into a vector of code regions to be + // associated with this BCB's coverage statement. + let code_regions = spans + .iter() + .map(|&span| make_code_region(source_map, file_name, span, body_span)) + .collect::<Vec<_>>(); inject_statement( self.mir_body, self.make_mir_coverage_kind(&counter_kind), self.bcb_leader_bb(bcb), - Some(code_region), + code_regions, ); } } - /// `inject_coverage_span_counters()` looped through the `CoverageSpan`s and injected the - /// counter from the `CoverageSpan`s `BasicCoverageBlock`, removing it from the BCB in the - /// process (via `take_counter()`). + /// At this point, any BCB with coverage counters has already had its counter injected + /// into MIR, and had its counter removed from `coverage_counters` (via `take_counter()`). /// /// Any other counter associated with a `BasicCoverageBlock`, or its incoming edge, but not - /// associated with a `CoverageSpan`, should only exist if the counter is an `Expression` + /// associated with a coverage span, should only exist if the counter is an `Expression` /// dependency (one of the expression operands). Collect them, and inject the additional /// counters into the MIR, without a reportable coverage span. fn inject_indirect_counters(&mut self) { @@ -303,7 +295,7 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { self.mir_body, self.make_mir_coverage_kind(&counter_kind), inject_to_bb, - None, + Vec::new(), ); } BcbCounter::Expression { .. } => inject_intermediate_expression( @@ -368,20 +360,14 @@ fn inject_statement( mir_body: &mut mir::Body<'_>, counter_kind: CoverageKind, bb: BasicBlock, - some_code_region: Option<CodeRegion>, + code_regions: Vec<CodeRegion>, ) { - debug!( - " injecting statement {:?} for {:?} at code region: {:?}", - counter_kind, bb, some_code_region - ); + debug!(" injecting statement {counter_kind:?} for {bb:?} at code regions: {code_regions:?}"); let data = &mut mir_body[bb]; let source_info = data.terminator().source_info; let statement = Statement { source_info, - kind: StatementKind::Coverage(Box::new(Coverage { - kind: counter_kind, - code_region: some_code_region, - })), + kind: StatementKind::Coverage(Box::new(Coverage { kind: counter_kind, code_regions })), }; data.statements.insert(0, statement); } @@ -395,7 +381,10 @@ fn inject_intermediate_expression(mir_body: &mut mir::Body<'_>, expression: Cove let source_info = data.terminator().source_info; let statement = Statement { source_info, - kind: StatementKind::Coverage(Box::new(Coverage { kind: expression, code_region: None })), + kind: StatementKind::Coverage(Box::new(Coverage { + kind: expression, + code_regions: Vec::new(), + })), }; data.statements.push(statement); } diff --git a/compiler/rustc_mir_transform/src/coverage/query.rs b/compiler/rustc_mir_transform/src/coverage/query.rs index 56365c5d474..2c0164e765c 100644 --- a/compiler/rustc_mir_transform/src/coverage/query.rs +++ b/compiler/rustc_mir_transform/src/coverage/query.rs @@ -93,8 +93,8 @@ fn coverageinfo<'tcx>(tcx: TyCtxt<'tcx>, instance_def: ty::InstanceDef<'tcx>) -> fn covered_code_regions(tcx: TyCtxt<'_>, def_id: DefId) -> Vec<&CodeRegion> { let body = mir_body(tcx, def_id); all_coverage_in_mir_body(body) - // Not all coverage statements have an attached code region. - .filter_map(|coverage| coverage.code_region.as_ref()) + // Coverage statements have a list of code regions (possibly empty). + .flat_map(|coverage| coverage.code_regions.as_slice()) .collect() } diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs index 5b24fa10bea..dd87694f97c 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans.rs @@ -1,6 +1,7 @@ use super::graph::{BasicCoverageBlock, BasicCoverageBlockData, CoverageGraph, START_BCB}; use rustc_data_structures::graph::WithNumNodes; +use rustc_index::IndexVec; use rustc_middle::mir::{ self, AggregateKind, BasicBlock, FakeReadCause, Rvalue, Statement, StatementKind, Terminator, TerminatorKind, @@ -10,6 +11,48 @@ use rustc_span::{BytePos, ExpnKind, MacroKind, Span, Symbol}; use std::cell::OnceCell; +pub(super) struct CoverageSpans { + /// Map from BCBs to their list of coverage spans. + bcb_to_spans: IndexVec<BasicCoverageBlock, Vec<Span>>, +} + +impl CoverageSpans { + pub(super) fn generate_coverage_spans( + mir_body: &mir::Body<'_>, + fn_sig_span: Span, + body_span: Span, + basic_coverage_blocks: &CoverageGraph, + ) -> Self { + let coverage_spans = CoverageSpansGenerator::generate_coverage_spans( + mir_body, + fn_sig_span, + body_span, + basic_coverage_blocks, + ); + + // Group the coverage spans by BCB, with the BCBs in sorted order. + let mut bcb_to_spans = IndexVec::from_elem_n(Vec::new(), basic_coverage_blocks.num_nodes()); + for CoverageSpan { bcb, span, .. } in coverage_spans { + bcb_to_spans[bcb].push(span); + } + + Self { bcb_to_spans } + } + + pub(super) fn bcb_has_coverage_spans(&self, bcb: BasicCoverageBlock) -> bool { + !self.bcb_to_spans[bcb].is_empty() + } + + pub(super) fn bcbs_with_coverage_spans( + &self, + ) -> impl Iterator<Item = (BasicCoverageBlock, &[Span])> { + self.bcb_to_spans.iter_enumerated().filter_map(|(bcb, spans)| { + // Only yield BCBs that have at least one coverage span. + (!spans.is_empty()).then_some((bcb, spans.as_slice())) + }) + } +} + #[derive(Debug, Copy, Clone)] pub(super) enum CoverageStatement { Statement(BasicBlock, Span, usize), @@ -35,7 +78,7 @@ impl CoverageStatement { /// or is subsumed by the `Span` associated with this `CoverageSpan`, and it's `BasicBlock` /// `dominates()` the `BasicBlock`s in this `CoverageSpan`. #[derive(Debug, Clone)] -pub(super) struct CoverageSpan { +struct CoverageSpan { pub span: Span, pub expn_span: Span, pub current_macro_or_none: OnceCell<Option<Symbol>>, @@ -162,7 +205,7 @@ impl CoverageSpan { /// * Merge spans that represent continuous (both in source code and control flow), non-branching /// execution /// * Carve out (leave uncovered) any span that will be counted by another MIR (notably, closures) -pub struct CoverageSpans<'a, 'tcx> { +struct CoverageSpansGenerator<'a, 'tcx> { /// The MIR, used to look up `BasicBlockData`. mir_body: &'a mir::Body<'tcx>, @@ -218,7 +261,7 @@ pub struct CoverageSpans<'a, 'tcx> { refined_spans: Vec<CoverageSpan>, } -impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { +impl<'a, 'tcx> CoverageSpansGenerator<'a, 'tcx> { /// Generate a minimal set of `CoverageSpan`s, each representing a contiguous code region to be /// counted. /// @@ -246,7 +289,7 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { body_span: Span, basic_coverage_blocks: &'a CoverageGraph, ) -> Vec<CoverageSpan> { - let mut coverage_spans = CoverageSpans { + let mut coverage_spans = Self { mir_body, fn_sig_span, body_span, @@ -734,7 +777,7 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { /// If the MIR `Statement` has a span contributive to computing coverage spans, /// return it; otherwise return `None`. -pub(super) fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span> { +fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span> { match statement.kind { // These statements have spans that are often outside the scope of the executed source code // for their parent `BasicBlock`. @@ -763,7 +806,7 @@ pub(super) fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span> // and `_1` is the `Place` for `somenum`. // // If and when the Issue is resolved, remove this special case match pattern: - StatementKind::FakeRead(box (cause, _)) if cause == FakeReadCause::ForGuardBinding => None, + StatementKind::FakeRead(box (FakeReadCause::ForGuardBinding, _)) => None, // Retain spans from all other statements StatementKind::FakeRead(box (_, _)) // Not including `ForGuardBinding` @@ -781,7 +824,7 @@ pub(super) fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span> /// If the MIR `Terminator` has a span contributive to computing coverage spans, /// return it; otherwise return `None`. -pub(super) fn filtered_terminator_span(terminator: &Terminator<'_>) -> Option<Span> { +fn filtered_terminator_span(terminator: &Terminator<'_>) -> Option<Span> { match terminator.kind { // These terminators have spans that don't positively contribute to computing a reasonable // span of actually executed source code. (For example, SwitchInt terminators extracted from @@ -828,7 +871,7 @@ pub(super) fn filtered_terminator_span(terminator: &Terminator<'_>) -> Option<Sp /// [^1]Expansions result from Rust syntax including macros, syntactic sugar, /// etc.). #[inline] -pub(super) fn function_source_span(span: Span, body_span: Span) -> Span { +fn function_source_span(span: Span, body_span: Span) -> Span { let original_span = original_sp(span, body_span).with_ctxt(body_span.ctxt()); if body_span.contains(original_span) { original_span } else { body_span } } diff --git a/compiler/rustc_mir_transform/src/coverage/tests.rs b/compiler/rustc_mir_transform/src/coverage/tests.rs index 4a066ed3abd..7476d3ce927 100644 --- a/compiler/rustc_mir_transform/src/coverage/tests.rs +++ b/compiler/rustc_mir_transform/src/coverage/tests.rs @@ -25,8 +25,7 @@ //! to: `rustc_span::create_default_session_globals_then(|| { test_here(); })`. use super::counters; -use super::graph; -use super::spans; +use super::graph::{self, BasicCoverageBlock}; use coverage_test_macros::let_bcb; @@ -644,39 +643,18 @@ fn test_traverse_coverage_with_loops() { ); } -fn synthesize_body_span_from_terminators(mir_body: &Body<'_>) -> Span { - let mut some_span: Option<Span> = None; - for (_, data) in mir_body.basic_blocks.iter_enumerated() { - let term_span = data.terminator().source_info.span; - if let Some(span) = some_span.as_mut() { - *span = span.to(term_span); - } else { - some_span = Some(term_span) - } - } - some_span.expect("body must have at least one BasicBlock") -} - #[test] fn test_make_bcb_counters() { rustc_span::create_default_session_globals_then(|| { let mir_body = goto_switchint(); - let body_span = synthesize_body_span_from_terminators(&mir_body); let mut basic_coverage_blocks = graph::CoverageGraph::from_mir(&mir_body); - let mut coverage_spans = Vec::new(); - for (bcb, data) in basic_coverage_blocks.iter_enumerated() { - if let Some(span) = spans::filtered_terminator_span(data.terminator(&mir_body)) { - coverage_spans.push(spans::CoverageSpan::for_terminator( - spans::function_source_span(span, body_span), - span, - bcb, - data.last_bb(), - )); - } - } + // Historically this test would use `spans` internals to set up fake + // coverage spans for BCBs 1 and 2. Now we skip that step and just tell + // BCB counter construction that those BCBs have spans. + let bcb_has_coverage_spans = |bcb: BasicCoverageBlock| (1..=2).contains(&bcb.as_usize()); let mut coverage_counters = counters::CoverageCounters::new(&basic_coverage_blocks); coverage_counters - .make_bcb_counters(&mut basic_coverage_blocks, &coverage_spans) + .make_bcb_counters(&mut basic_coverage_blocks, bcb_has_coverage_spans) .expect("should be Ok"); assert_eq!(coverage_counters.intermediate_expressions.len(), 0); diff --git a/compiler/rustc_mir_transform/src/deref_separator.rs b/compiler/rustc_mir_transform/src/deref_separator.rs index 95898b5b73c..42be7457018 100644 --- a/compiler/rustc_mir_transform/src/deref_separator.rs +++ b/compiler/rustc_mir_transform/src/deref_separator.rs @@ -37,7 +37,7 @@ impl<'a, 'tcx> MutVisitor<'tcx> for DerefChecker<'a, 'tcx> { for (idx, (p_ref, p_elem)) in place.iter_projections().enumerate() { if !p_ref.projection.is_empty() && p_elem == ProjectionElem::Deref { let ty = p_ref.ty(self.local_decls, self.tcx).ty; - let temp = self.patcher.new_internal_with_info( + let temp = self.patcher.new_local_with_info( ty, self.local_decls[p_ref.local].source_info.span, LocalInfo::DerefTemp, diff --git a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs index e51f771e00d..1c917a85c03 100644 --- a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs +++ b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs @@ -69,7 +69,7 @@ impl<'tcx, 'a> MutVisitor<'tcx> for ElaborateBoxDerefVisitor<'tcx, 'a> { let (unique_ty, nonnull_ty, ptr_ty) = build_ptr_tys(tcx, base_ty.boxed_ty(), self.unique_did, self.nonnull_did); - let ptr_local = self.patch.new_internal(ptr_ty, source_info.span); + let ptr_local = self.patch.new_temp(ptr_ty, source_info.span); self.patch.add_assign( location, diff --git a/compiler/rustc_mir_transform/src/elaborate_drops.rs b/compiler/rustc_mir_transform/src/elaborate_drops.rs index 6a89d067275..d18fdaaf22f 100644 --- a/compiler/rustc_mir_transform/src/elaborate_drops.rs +++ b/compiler/rustc_mir_transform/src/elaborate_drops.rs @@ -170,6 +170,7 @@ impl<'a, 'tcx> DropElaborator<'a, 'tcx> for Elaborator<'a, '_, 'tcx> { self.ctxt.param_env() } + #[instrument(level = "debug", skip(self), ret)] fn drop_style(&self, path: Self::Path, mode: DropFlagMode) -> DropStyle { let ((maybe_live, maybe_dead), multipart) = match mode { DropFlagMode::Shallow => (self.ctxt.init_data.maybe_live_dead(path), false), @@ -270,7 +271,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { let tcx = self.tcx; let patch = &mut self.patch; debug!("create_drop_flag({:?})", self.body.span); - self.drop_flags[index].get_or_insert_with(|| patch.new_internal(tcx.types.bool, span)); + self.drop_flags[index].get_or_insert_with(|| patch.new_temp(tcx.types.bool, span)); } fn drop_flag(&mut self, index: MovePathIndex) -> Option<Place<'tcx>> { diff --git a/compiler/rustc_mir_transform/src/generator.rs b/compiler/rustc_mir_transform/src/generator.rs index 8a807d786a5..c16f07a453c 100644 --- a/compiler/rustc_mir_transform/src/generator.rs +++ b/compiler/rustc_mir_transform/src/generator.rs @@ -321,7 +321,7 @@ impl<'tcx> TransformVisitor<'tcx> { // Create a statement which reads the discriminant into a temporary fn get_discr(&self, body: &mut Body<'tcx>) -> (Statement<'tcx>, Place<'tcx>) { - let temp_decl = LocalDecl::new(self.discr_ty, body.span).internal(); + let temp_decl = LocalDecl::new(self.discr_ty, body.span); let local_decls_len = body.local_decls.push(temp_decl); let temp = Place::from(local_decls_len); diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs index 449bade3322..56bdc5a171a 100644 --- a/compiler/rustc_mir_transform/src/gvn.rs +++ b/compiler/rustc_mir_transform/src/gvn.rs @@ -306,6 +306,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> { } ProjectionElem::Downcast(name, index) => ProjectionElem::Downcast(name, index), ProjectionElem::OpaqueCast(ty) => ProjectionElem::OpaqueCast(ty), + ProjectionElem::Subtype(ty) => ProjectionElem::Subtype(ty), }; value = self.insert(Value::Projection(value, proj)); } diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index ebd61f8ad95..32dfb743905 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -218,7 +218,13 @@ impl<'tcx> Inliner<'tcx> { // Normally, this shouldn't be required, but trait normalization failure can create a // validation ICE. let output_type = callee_body.return_ty(); - if !util::is_subtype(self.tcx, self.param_env, output_type, destination_ty) { + if !util::relate_types( + self.tcx, + self.param_env, + ty::Variance::Covariant, + output_type, + destination_ty, + ) { trace!(?output_type, ?destination_ty); return Err("failed to normalize return type"); } @@ -248,7 +254,13 @@ impl<'tcx> Inliner<'tcx> { self_arg_ty.into_iter().chain(arg_tuple_tys).zip(callee_body.args_iter()) { let input_type = callee_body.local_decls[input].ty; - if !util::is_subtype(self.tcx, self.param_env, input_type, arg_ty) { + if !util::relate_types( + self.tcx, + self.param_env, + ty::Variance::Covariant, + input_type, + arg_ty, + ) { trace!(?arg_ty, ?input_type); return Err("failed to normalize tuple argument type"); } @@ -257,7 +269,13 @@ impl<'tcx> Inliner<'tcx> { for (arg, input) in args.iter().zip(callee_body.args_iter()) { let input_type = callee_body.local_decls[input].ty; let arg_ty = arg.ty(&caller_body.local_decls, self.tcx); - if !util::is_subtype(self.tcx, self.param_env, input_type, arg_ty) { + if !util::relate_types( + self.tcx, + self.param_env, + ty::Variance::Covariant, + input_type, + arg_ty, + ) { trace!(?arg_ty, ?input_type); return Err("failed to normalize argument type"); } @@ -598,9 +616,7 @@ impl<'tcx> Inliner<'tcx> { // If there are any locals without storage markers, give them storage only for the // duration of the call. for local in callee_body.vars_and_temps_iter() { - if !callee_body.local_decls[local].internal - && integrator.always_live_locals.contains(local) - { + if integrator.always_live_locals.contains(local) { let new_local = integrator.map_local(local); caller_body[callsite.block].statements.push(Statement { source_info: callsite.source_info, @@ -623,9 +639,7 @@ impl<'tcx> Inliner<'tcx> { n += 1; } for local in callee_body.vars_and_temps_iter().rev() { - if !callee_body.local_decls[local].internal - && integrator.always_live_locals.contains(local) - { + if integrator.always_live_locals.contains(local) { let new_local = integrator.map_local(local); caller_body[block].statements.push(Statement { source_info: callsite.source_info, diff --git a/compiler/rustc_mir_transform/src/large_enums.rs b/compiler/rustc_mir_transform/src/large_enums.rs index 4eee45f8d00..886ff760481 100644 --- a/compiler/rustc_mir_transform/src/large_enums.rs +++ b/compiler/rustc_mir_transform/src/large_enums.rs @@ -54,11 +54,8 @@ impl EnumSizeOpt { let layout = tcx.layout_of(param_env.and(ty)).ok()?; let variants = match &layout.variants { Variants::Single { .. } => return None, - Variants::Multiple { tag_encoding, .. } - if matches!(tag_encoding, TagEncoding::Niche { .. }) => - { - return None; - } + Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, .. } => return None, + Variants::Multiple { variants, .. } if variants.len() <= 1 => return None, Variants::Multiple { variants, .. } => variants, }; diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 9e4bc456d51..22381844d6d 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -54,6 +54,7 @@ mod check_packed_ref; pub mod check_unsafety; mod remove_place_mention; // This pass is public to allow external drivers to perform MIR cleanup +mod add_subtyping_projections; pub mod cleanup_post_borrowck; mod const_debuginfo; mod const_goto; @@ -466,6 +467,7 @@ pub fn run_analysis_to_runtime_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<' /// After this series of passes, no lifetime analysis based on borrowing can be done. fn run_analysis_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let passes: &[&dyn MirPass<'tcx>] = &[ + &add_subtyping_projections::Subtyper, &cleanup_post_borrowck::CleanupPostBorrowck, &remove_noop_landing_pads::RemoveNoopLandingPads, &simplify::SimplifyCfg::EarlyOpt, @@ -480,6 +482,7 @@ fn run_runtime_lowering_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let passes: &[&dyn MirPass<'tcx>] = &[ // These next passes must be executed together &add_call_guards::CriticalCallEdges, + &reveal_all::RevealAll, // has to be done before drop elaboration, since we need to drop opaque types, too. &elaborate_drops::ElaborateDrops, // This will remove extraneous landing pads which are no longer // necessary as well as well as forcing any call in a non-unwinding @@ -526,7 +529,6 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { body, &[ &check_alignment::CheckAlignment, - &reveal_all::RevealAll, // has to be done before inlining, since inlined code is in RevealAll mode. &lower_slice_len::LowerSliceLenCalls, // has to be done before inlining, otherwise actual call will be almost always inlined. Also simple, so can just do first &unreachable_prop::UnreachablePropagation, &uninhabited_enum_branching::UninhabitedEnumBranching, diff --git a/compiler/rustc_mir_transform/src/reveal_all.rs b/compiler/rustc_mir_transform/src/reveal_all.rs index 065793348e4..55f1eac6f84 100644 --- a/compiler/rustc_mir_transform/src/reveal_all.rs +++ b/compiler/rustc_mir_transform/src/reveal_all.rs @@ -8,16 +8,7 @@ use rustc_middle::ty::{self, Ty, TyCtxt}; pub struct RevealAll; impl<'tcx> MirPass<'tcx> for RevealAll { - fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - sess.mir_opt_level() >= 3 || super::inline::Inline.is_enabled(sess) - } - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - // Do not apply this transformation to generators. - if body.generator.is_some() { - return; - } - let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); RevealAllVisitor { tcx, param_env }.visit_body_preserves_cfg(body); } @@ -35,6 +26,29 @@ impl<'tcx> MutVisitor<'tcx> for RevealAllVisitor<'tcx> { } #[inline] + fn visit_place( + &mut self, + place: &mut Place<'tcx>, + _context: PlaceContext, + _location: Location, + ) { + // Performance optimization: don't reintern if there is no `OpaqueCast` to remove. + if place.projection.iter().all(|elem| !matches!(elem, ProjectionElem::OpaqueCast(_))) { + return; + } + // `OpaqueCast` projections are only needed if there are opaque types on which projections are performed. + // After the `RevealAll` pass, all opaque types are replaced with their hidden types, so we don't need these + // projections anymore. + place.projection = self.tcx.mk_place_elems( + &place + .projection + .into_iter() + .filter(|elem| !matches!(elem, ProjectionElem::OpaqueCast(_))) + .collect::<Vec<_>>(), + ); + } + + #[inline] fn visit_constant(&mut self, constant: &mut ConstOperand<'tcx>, _: Location) { // We have to use `try_normalize_erasing_regions` here, since it's // possible that we visit impossible-to-satisfy where clauses here, diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index 2795cf15702..73dae044355 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -441,21 +441,23 @@ fn save_unreachable_coverage( let dead_block = &basic_blocks[dead_block]; for statement in &dead_block.statements { let StatementKind::Coverage(coverage) = &statement.kind else { continue }; - let Some(code_region) = &coverage.code_region else { continue }; + if coverage.code_regions.is_empty() { + continue; + }; let instance = statement.source_info.scope.inlined_instance(source_scopes); if live.contains(&instance) { - retained_coverage.push((statement.source_info, code_region.clone())); + retained_coverage.push((statement.source_info, coverage.code_regions.clone())); } } } let start_block = &mut basic_blocks[START_BLOCK]; start_block.statements.extend(retained_coverage.into_iter().map( - |(source_info, code_region)| Statement { + |(source_info, code_regions)| Statement { source_info, kind: StatementKind::Coverage(Box::new(Coverage { kind: CoverageKind::Unreachable, - code_region: Some(code_region), + code_regions, })), }, )); diff --git a/compiler/rustc_mir_transform/src/ssa.rs b/compiler/rustc_mir_transform/src/ssa.rs index 3a675752fba..af9514ed6bb 100644 --- a/compiler/rustc_mir_transform/src/ssa.rs +++ b/compiler/rustc_mir_transform/src/ssa.rs @@ -78,14 +78,10 @@ impl SsaLocals { visitor.assignments[local] = Set1::One(LocationExtended::Arg); } - if body.basic_blocks.len() > 2 { - for (bb, data) in traversal::reverse_postorder(body) { - visitor.visit_basic_block_data(bb, data); - } - } else { - for (bb, data) in body.basic_blocks.iter_enumerated() { - visitor.visit_basic_block_data(bb, data); - } + // For SSA assignments, a RPO visit will see the assignment before it sees any use. + // We only visit reachable nodes: computing `dominates` on an unreachable node ICEs. + for (bb, data) in traversal::reverse_postorder(body) { + visitor.visit_basic_block_data(bb, data); } for var_debug_info in &body.var_debug_info { |