diff options
Diffstat (limited to 'compiler/rustc_mir_transform/src')
| -rw-r--r-- | compiler/rustc_mir_transform/src/coroutine.rs | 42 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/cost_checker.rs | 7 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/coverage/graph.rs | 9 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/coverage/mappings.rs | 22 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/coverage/mod.rs | 80 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/coverage/spans.rs | 7 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs | 44 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/dest_prop.rs | 6 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/elaborate_drops.rs | 20 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/gvn.rs | 14 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/inline.rs | 15 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/jump_threading.rs | 1 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/known_panics_lint.rs | 1 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/mentioned_items.rs | 2 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs | 1 | ||||
| -rw-r--r-- | compiler/rustc_mir_transform/src/validate.rs | 77 |
16 files changed, 223 insertions, 125 deletions
diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs index 4c00038365b..261dcd52d71 100644 --- a/compiler/rustc_mir_transform/src/coroutine.rs +++ b/compiler/rustc_mir_transform/src/coroutine.rs @@ -58,7 +58,7 @@ use crate::deref_separator::deref_finder; use crate::errors; use crate::pass_manager as pm; use crate::simplify; -use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use rustc_data_structures::fx::FxHashSet; use rustc_errors::pluralize; use rustc_hir as hir; use rustc_hir::lang_items::LangItem; @@ -81,8 +81,8 @@ use rustc_span::symbol::sym; use rustc_span::Span; use rustc_target::abi::{FieldIdx, VariantIdx}; use rustc_target::spec::PanicStrategy; +use rustc_trait_selection::error_reporting::traits::TypeErrCtxtExt as _; use rustc_trait_selection::infer::TyCtxtInferExt as _; -use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt as _; use rustc_trait_selection::traits::ObligationCtxt; use rustc_trait_selection::traits::{ObligationCause, ObligationCauseCode}; use std::{iter, ops}; @@ -233,8 +233,7 @@ struct TransformVisitor<'tcx> { discr_ty: Ty<'tcx>, // Mapping from Local to (type of local, coroutine struct index) - // FIXME(eddyb) This should use `IndexVec<Local, Option<_>>`. - remap: FxHashMap<Local, (Ty<'tcx>, VariantIdx, FieldIdx)>, + remap: IndexVec<Local, Option<(Ty<'tcx>, VariantIdx, FieldIdx)>>, // A map from a suspension point in a block to the locals which have live storage at that point storage_liveness: IndexVec<BasicBlock, Option<BitSet<Local>>>, @@ -482,7 +481,7 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { } fn visit_local(&mut self, local: &mut Local, _: PlaceContext, _: Location) { - assert_eq!(self.remap.get(local), None); + assert!(!self.remap.contains(*local)); } fn visit_place( @@ -492,7 +491,7 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { _location: Location, ) { // Replace an Local in the remap with a coroutine struct access - if let Some(&(ty, variant_index, idx)) = self.remap.get(&place.local) { + if let Some(&Some((ty, variant_index, idx))) = self.remap.get(place.local) { replace_base(place, self.make_field(variant_index, idx, ty), self.tcx); } } @@ -501,7 +500,7 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { // Remove StorageLive and StorageDead statements for remapped locals data.retain_statements(|s| match s.kind { StatementKind::StorageLive(l) | StatementKind::StorageDead(l) => { - !self.remap.contains_key(&l) + !self.remap.contains(l) } _ => true, }); @@ -526,13 +525,9 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { // The resume arg target location might itself be remapped if its base local is // live across a yield. - let resume_arg = - if let Some(&(ty, variant, idx)) = self.remap.get(&resume_arg.local) { - replace_base(&mut resume_arg, self.make_field(variant, idx, ty), self.tcx); - resume_arg - } else { - resume_arg - }; + if let Some(&Some((ty, variant, idx))) = self.remap.get(resume_arg.local) { + replace_base(&mut resume_arg, self.make_field(variant, idx, ty), self.tcx); + } let storage_liveness: GrowableBitSet<Local> = self.storage_liveness[block].clone().unwrap().into(); @@ -540,7 +535,7 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { for i in 0..self.always_live_locals.domain_size() { let l = Local::new(i); let needs_storage_dead = storage_liveness.contains(l) - && !self.remap.contains_key(&l) + && !self.remap.contains(l) && !self.always_live_locals.contains(l); if needs_storage_dead { data.statements @@ -1034,7 +1029,7 @@ fn compute_layout<'tcx>( liveness: LivenessInfo, body: &Body<'tcx>, ) -> ( - FxHashMap<Local, (Ty<'tcx>, VariantIdx, FieldIdx)>, + IndexVec<Local, Option<(Ty<'tcx>, VariantIdx, FieldIdx)>>, CoroutineLayout<'tcx>, IndexVec<BasicBlock, Option<BitSet<Local>>>, ) { @@ -1095,7 +1090,7 @@ fn compute_layout<'tcx>( // Create a map from local indices to coroutine struct indices. let mut variant_fields: IndexVec<VariantIdx, IndexVec<FieldIdx, CoroutineSavedLocal>> = iter::repeat(IndexVec::new()).take(RESERVED_VARIANTS).collect(); - let mut remap = FxHashMap::default(); + let mut remap = IndexVec::from_elem_n(None, saved_locals.domain_size()); for (suspension_point_idx, live_locals) in live_locals_at_suspension_points.iter().enumerate() { let variant_index = VariantIdx::from(RESERVED_VARIANTS + suspension_point_idx); let mut fields = IndexVec::new(); @@ -1106,7 +1101,7 @@ fn compute_layout<'tcx>( // around inside coroutines, so it doesn't matter which variant // index we access them by. let idx = FieldIdx::from_usize(idx); - remap.entry(locals[saved_local]).or_insert((tys[saved_local].ty, variant_index, idx)); + remap[locals[saved_local]] = Some((tys[saved_local].ty, variant_index, idx)); } variant_fields.push(fields); variant_source_info.push(source_info_at_suspension_points[suspension_point_idx]); @@ -1118,7 +1113,9 @@ fn compute_layout<'tcx>( for var in &body.var_debug_info { let VarDebugInfoContents::Place(place) = &var.value else { continue }; let Some(local) = place.as_local() else { continue }; - let Some(&(_, variant, field)) = remap.get(&local) else { continue }; + let Some(&Some((_, variant, field))) = remap.get(local) else { + continue; + }; let saved_local = variant_fields[variant][field]; field_names.get_or_insert_with(saved_local, || var.name); @@ -1370,6 +1367,10 @@ fn can_unwind<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> bool { | TerminatorKind::Call { .. } | TerminatorKind::InlineAsm { .. } | TerminatorKind::Assert { .. } => return true, + + TerminatorKind::TailCall { .. } => { + unreachable!("tail calls can't be present in generators") + } } } @@ -1521,7 +1522,7 @@ fn create_cases<'tcx>( for i in 0..(body.local_decls.len()) { let l = Local::new(i); let needs_storage_live = point.storage_liveness.contains(l) - && !transform.remap.contains_key(&l) + && !transform.remap.contains(l) && !transform.always_live_locals.contains(l); if needs_storage_live { statements @@ -1919,6 +1920,7 @@ impl<'tcx> Visitor<'tcx> for EnsureCoroutineFieldAssignmentsNeverAlias<'_> { | TerminatorKind::UnwindResume | TerminatorKind::UnwindTerminate(_) | TerminatorKind::Return + | TerminatorKind::TailCall { .. } | TerminatorKind::Unreachable | TerminatorKind::Drop { .. } | TerminatorKind::Assert { .. } diff --git a/compiler/rustc_mir_transform/src/cost_checker.rs b/compiler/rustc_mir_transform/src/cost_checker.rs index 3333bebff3a..a1c1422912e 100644 --- a/compiler/rustc_mir_transform/src/cost_checker.rs +++ b/compiler/rustc_mir_transform/src/cost_checker.rs @@ -40,7 +40,9 @@ impl<'b, 'tcx> CostChecker<'b, 'tcx> { fn is_call_like(bbd: &BasicBlockData<'_>) -> bool { use TerminatorKind::*; match bbd.terminator().kind { - Call { .. } | Drop { .. } | Assert { .. } | InlineAsm { .. } => true, + Call { .. } | TailCall { .. } | Drop { .. } | Assert { .. } | InlineAsm { .. } => { + true + } Goto { .. } | SwitchInt { .. } @@ -137,6 +139,9 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { self.penalty += LANDINGPAD_PENALTY; } } + TerminatorKind::TailCall { .. } => { + self.penalty += CALL_PENALTY; + } TerminatorKind::SwitchInt { discr, targets } => { if discr.constant().is_some() { // Not only will this become a `Goto`, but likely other diff --git a/compiler/rustc_mir_transform/src/coverage/graph.rs b/compiler/rustc_mir_transform/src/coverage/graph.rs index 360dccb240d..83fb9ff9743 100644 --- a/compiler/rustc_mir_transform/src/coverage/graph.rs +++ b/compiler/rustc_mir_transform/src/coverage/graph.rs @@ -358,9 +358,12 @@ fn bcb_filtered_successors<'a, 'tcx>(terminator: &'a Terminator<'tcx>) -> Covera } // These terminators have no coverage-relevant successors. - CoroutineDrop | Return | Unreachable | UnwindResume | UnwindTerminate(_) => { - CoverageSuccessors::NotChainable(&[]) - } + CoroutineDrop + | Return + | TailCall { .. } + | Unreachable + | UnwindResume + | UnwindTerminate(_) => CoverageSuccessors::NotChainable(&[]), } } diff --git a/compiler/rustc_mir_transform/src/coverage/mappings.rs b/compiler/rustc_mir_transform/src/coverage/mappings.rs index 235992ac547..25297245172 100644 --- a/compiler/rustc_mir_transform/src/coverage/mappings.rs +++ b/compiler/rustc_mir_transform/src/coverage/mappings.rs @@ -3,7 +3,9 @@ use std::collections::BTreeSet; use rustc_data_structures::graph::DirectedGraph; use rustc_index::bit_set::BitSet; use rustc_index::IndexVec; -use rustc_middle::mir::coverage::{BlockMarkerId, BranchSpan, ConditionInfo, CoverageKind}; +use rustc_middle::mir::coverage::{ + BlockMarkerId, BranchSpan, ConditionInfo, CoverageInfoHi, CoverageKind, +}; use rustc_middle::mir::{self, BasicBlock, StatementKind}; use rustc_middle::ty::TyCtxt; use rustc_span::Span; @@ -157,12 +159,12 @@ impl ExtractedMappings { } fn resolve_block_markers( - branch_info: &mir::coverage::BranchInfo, + coverage_info_hi: &CoverageInfoHi, mir_body: &mir::Body<'_>, ) -> IndexVec<BlockMarkerId, Option<BasicBlock>> { let mut block_markers = IndexVec::<BlockMarkerId, Option<BasicBlock>>::from_elem_n( None, - branch_info.num_block_markers, + coverage_info_hi.num_block_markers, ); // Fill out the mapping from block marker IDs to their enclosing blocks. @@ -188,11 +190,11 @@ pub(super) fn extract_branch_pairs( hir_info: &ExtractedHirInfo, basic_coverage_blocks: &CoverageGraph, ) -> Vec<BranchPair> { - let Some(branch_info) = mir_body.coverage_branch_info.as_deref() else { return vec![] }; + let Some(coverage_info_hi) = mir_body.coverage_info_hi.as_deref() else { return vec![] }; - let block_markers = resolve_block_markers(branch_info, mir_body); + let block_markers = resolve_block_markers(coverage_info_hi, mir_body); - branch_info + coverage_info_hi .branch_spans .iter() .filter_map(|&BranchSpan { span: raw_span, true_marker, false_marker }| { @@ -222,9 +224,9 @@ pub(super) fn extract_mcdc_mappings( mcdc_branches: &mut impl Extend<MCDCBranch>, mcdc_decisions: &mut impl Extend<MCDCDecision>, ) { - let Some(branch_info) = mir_body.coverage_branch_info.as_deref() else { return }; + let Some(coverage_info_hi) = mir_body.coverage_info_hi.as_deref() else { return }; - let block_markers = resolve_block_markers(branch_info, mir_body); + let block_markers = resolve_block_markers(coverage_info_hi, mir_body); let bcb_from_marker = |marker: BlockMarkerId| basic_coverage_blocks.bcb_from_bb(block_markers[marker]?); @@ -243,7 +245,7 @@ pub(super) fn extract_mcdc_mappings( Some((span, true_bcb, false_bcb)) }; - mcdc_branches.extend(branch_info.mcdc_branch_spans.iter().filter_map( + mcdc_branches.extend(coverage_info_hi.mcdc_branch_spans.iter().filter_map( |&mir::coverage::MCDCBranchSpan { span: raw_span, condition_info, @@ -257,7 +259,7 @@ pub(super) fn extract_mcdc_mappings( }, )); - mcdc_decisions.extend(branch_info.mcdc_decision_spans.iter().filter_map( + mcdc_decisions.extend(coverage_info_hi.mcdc_decision_spans.iter().filter_map( |decision: &mir::coverage::MCDCDecisionSpan| { let span = unexpand_into_body_span(decision.span, body_span)?; diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs index d55bde311c1..2efca40d180 100644 --- a/compiler/rustc_mir_transform/src/coverage/mod.rs +++ b/compiler/rustc_mir_transform/src/coverage/mod.rs @@ -8,6 +8,10 @@ mod spans; mod tests; mod unexpand; +use rustc_hir as hir; +use rustc_hir::intravisit::{walk_expr, Visitor}; +use rustc_middle::hir::map::Map; +use rustc_middle::hir::nested_filter; use rustc_middle::mir::coverage::{ CodeRegion, CoverageKind, DecisionInfo, FunctionCoverageInfo, Mapping, MappingKind, }; @@ -465,6 +469,9 @@ struct ExtractedHirInfo { /// Must have the same context and filename as the body span. fn_sig_span_extended: Option<Span>, body_span: Span, + /// "Holes" are regions within the body span that should not be included in + /// coverage spans for this function (e.g. closures and nested items). + hole_spans: Vec<Span>, } fn extract_hir_info<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> ExtractedHirInfo { @@ -480,7 +487,7 @@ fn extract_hir_info<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> ExtractedHir let mut body_span = hir_body.value.span; - use rustc_hir::{Closure, Expr, ExprKind, Node}; + use hir::{Closure, Expr, ExprKind, Node}; // Unexpand a closure's body span back to the context of its declaration. // This helps with closure bodies that consist of just a single bang-macro, // and also with closure bodies produced by async desugaring. @@ -507,11 +514,78 @@ fn extract_hir_info<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> ExtractedHir let function_source_hash = hash_mir_source(tcx, hir_body); - ExtractedHirInfo { function_source_hash, is_async_fn, fn_sig_span_extended, body_span } + let hole_spans = extract_hole_spans_from_hir(tcx, body_span, hir_body); + + ExtractedHirInfo { + function_source_hash, + is_async_fn, + fn_sig_span_extended, + body_span, + hole_spans, + } } -fn hash_mir_source<'tcx>(tcx: TyCtxt<'tcx>, hir_body: &'tcx rustc_hir::Body<'tcx>) -> u64 { +fn hash_mir_source<'tcx>(tcx: TyCtxt<'tcx>, hir_body: &'tcx hir::Body<'tcx>) -> u64 { // FIXME(cjgillot) Stop hashing HIR manually here. let owner = hir_body.id().hir_id.owner; tcx.hir_owner_nodes(owner).opt_hash_including_bodies.unwrap().to_smaller_hash().as_u64() } + +fn extract_hole_spans_from_hir<'tcx>( + tcx: TyCtxt<'tcx>, + body_span: Span, // Usually `hir_body.value.span`, but not always + hir_body: &hir::Body<'tcx>, +) -> Vec<Span> { + struct HolesVisitor<'hir, F> { + hir: Map<'hir>, + visit_hole_span: F, + } + + impl<'hir, F: FnMut(Span)> Visitor<'hir> for HolesVisitor<'hir, F> { + /// - We need `NestedFilter::INTRA = true` so that `visit_item` will be called. + /// - Bodies of nested items don't actually get visited, because of the + /// `visit_item` override. + /// - For nested bodies that are not part of an item, we do want to visit any + /// items contained within them. + type NestedFilter = nested_filter::All; + + fn nested_visit_map(&mut self) -> Self::Map { + self.hir + } + + fn visit_item(&mut self, item: &'hir hir::Item<'hir>) { + (self.visit_hole_span)(item.span); + // Having visited this item, we don't care about its children, + // so don't call `walk_item`. + } + + // We override `visit_expr` instead of the more specific expression + // visitors, so that we have direct access to the expression span. + fn visit_expr(&mut self, expr: &'hir hir::Expr<'hir>) { + match expr.kind { + hir::ExprKind::Closure(_) | hir::ExprKind::ConstBlock(_) => { + (self.visit_hole_span)(expr.span); + // Having visited this expression, we don't care about its + // children, so don't call `walk_expr`. + } + + // For other expressions, recursively visit as normal. + _ => walk_expr(self, expr), + } + } + } + + let mut hole_spans = vec![]; + let mut visitor = HolesVisitor { + hir: tcx.hir(), + visit_hole_span: |hole_span| { + // Discard any holes that aren't directly visible within the body span. + if body_span.contains(hole_span) && body_span.eq_ctxt(hole_span) { + hole_spans.push(hole_span); + } + }, + }; + + visitor.visit_body(hir_body); + hole_spans +} diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs index 7612c01c52e..dbc26a2808e 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans.rs @@ -8,7 +8,7 @@ use rustc_span::Span; use crate::coverage::graph::{BasicCoverageBlock, CoverageGraph}; use crate::coverage::mappings; use crate::coverage::spans::from_mir::{ - extract_covspans_and_holes_from_mir, ExtractedCovspans, Hole, SpanFromMir, + extract_covspans_from_mir, ExtractedCovspans, Hole, SpanFromMir, }; use crate::coverage::ExtractedHirInfo; @@ -20,8 +20,8 @@ pub(super) fn extract_refined_covspans( basic_coverage_blocks: &CoverageGraph, code_mappings: &mut impl Extend<mappings::CodeMapping>, ) { - let ExtractedCovspans { mut covspans, mut holes } = - extract_covspans_and_holes_from_mir(mir_body, hir_info, basic_coverage_blocks); + let ExtractedCovspans { mut covspans } = + extract_covspans_from_mir(mir_body, hir_info, basic_coverage_blocks); // First, perform the passes that need macro information. covspans.sort_by(|a, b| basic_coverage_blocks.cmp_in_dominator_order(a.bcb, b.bcb)); @@ -45,6 +45,7 @@ pub(super) fn extract_refined_covspans( covspans.dedup_by(|b, a| a.span.source_equal(b.span)); // Sort the holes, and merge overlapping/adjacent holes. + let mut holes = hir_info.hole_spans.iter().map(|&span| Hole { span }).collect::<Vec<_>>(); holes.sort_by(|a, b| compare_spans(a.span, b.span)); holes.dedup_by(|b, a| a.merge_if_overlapping_or_adjacent(b)); diff --git a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs index 2ca166929ee..32bd25bf4b9 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs @@ -1,8 +1,7 @@ use rustc_middle::bug; use rustc_middle::mir::coverage::CoverageKind; use rustc_middle::mir::{ - self, AggregateKind, FakeReadCause, Rvalue, Statement, StatementKind, Terminator, - TerminatorKind, + self, FakeReadCause, Statement, StatementKind, Terminator, TerminatorKind, }; use rustc_span::{Span, Symbol}; @@ -15,13 +14,12 @@ use crate::coverage::ExtractedHirInfo; pub(crate) struct ExtractedCovspans { pub(crate) covspans: Vec<SpanFromMir>, - pub(crate) holes: Vec<Hole>, } /// Traverses the MIR body to produce an initial collection of coverage-relevant /// spans, each associated with a node in the coverage graph (BCB) and possibly /// other metadata. -pub(crate) fn extract_covspans_and_holes_from_mir( +pub(crate) fn extract_covspans_from_mir( mir_body: &mir::Body<'_>, hir_info: &ExtractedHirInfo, basic_coverage_blocks: &CoverageGraph, @@ -29,21 +27,13 @@ pub(crate) fn extract_covspans_and_holes_from_mir( let &ExtractedHirInfo { body_span, .. } = hir_info; let mut covspans = vec![]; - let mut holes = vec![]; for (bcb, bcb_data) in basic_coverage_blocks.iter_enumerated() { - bcb_to_initial_coverage_spans( - mir_body, - body_span, - bcb, - bcb_data, - &mut covspans, - &mut holes, - ); + bcb_to_initial_coverage_spans(mir_body, body_span, bcb, bcb_data, &mut covspans); } // Only add the signature span if we found at least one span in the body. - if !covspans.is_empty() || !holes.is_empty() { + if !covspans.is_empty() { // If there is no usable signature span, add a fake one (before refinement) // to avoid an ugly gap between the body start and the first real span. // FIXME: Find a more principled way to solve this problem. @@ -51,7 +41,7 @@ pub(crate) fn extract_covspans_and_holes_from_mir( covspans.push(SpanFromMir::for_fn_sig(fn_sig_span)); } - ExtractedCovspans { covspans, holes } + ExtractedCovspans { covspans } } // Generate a set of coverage spans from the filtered set of `Statement`s and `Terminator`s of @@ -65,7 +55,6 @@ fn bcb_to_initial_coverage_spans<'a, 'tcx>( bcb: BasicCoverageBlock, bcb_data: &'a BasicCoverageBlockData, initial_covspans: &mut Vec<SpanFromMir>, - holes: &mut Vec<Hole>, ) { for &bb in &bcb_data.basic_blocks { let data = &mir_body[bb]; @@ -81,13 +70,7 @@ fn bcb_to_initial_coverage_spans<'a, 'tcx>( let expn_span = filtered_statement_span(statement)?; let (span, visible_macro) = unexpand(expn_span)?; - // A statement that looks like the assignment of a closure expression - // is treated as a "hole" span, to be carved out of other spans. - if is_closure_like(statement) { - holes.push(Hole { span }); - } else { - initial_covspans.push(SpanFromMir::new(span, visible_macro, bcb)); - } + initial_covspans.push(SpanFromMir::new(span, visible_macro, bcb)); Some(()) }; for statement in data.statements.iter() { @@ -105,18 +88,6 @@ fn bcb_to_initial_coverage_spans<'a, 'tcx>( } } -fn is_closure_like(statement: &Statement<'_>) -> bool { - match statement.kind { - StatementKind::Assign(box (_, Rvalue::Aggregate(box ref agg_kind, _))) => match agg_kind { - AggregateKind::Closure(_, _) - | AggregateKind::Coroutine(_, _) - | AggregateKind::CoroutineClosure(..) => true, - _ => false, - }, - _ => false, - } -} - /// If the MIR `Statement` has a span contributive to computing coverage spans, /// return it; otherwise return `None`. fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span> { @@ -193,7 +164,8 @@ fn filtered_terminator_span(terminator: &Terminator<'_>) -> Option<Span> { | TerminatorKind::Goto { .. } => None, // Call `func` operand can have a more specific span when part of a chain of calls - | TerminatorKind::Call { ref func, .. } => { + TerminatorKind::Call { ref func, .. } + | TerminatorKind::TailCall { ref func, .. } => { let mut span = terminator.source_info.span; if let mir::Operand::Constant(box constant) = func { if constant.span.lo() > span.lo() { diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index b1016c0867c..ab73a8af317 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -628,6 +628,12 @@ impl WriteInfo { self.add_operand(&arg.node); } } + TerminatorKind::TailCall { func, args, .. } => { + self.add_operand(func); + for arg in args { + self.add_operand(&arg.node); + } + } TerminatorKind::InlineAsm { operands, .. } => { for asm_operand in operands { match asm_operand { diff --git a/compiler/rustc_mir_transform/src/elaborate_drops.rs b/compiler/rustc_mir_transform/src/elaborate_drops.rs index fbbb8c5e472..25bebb0539a 100644 --- a/compiler/rustc_mir_transform/src/elaborate_drops.rs +++ b/compiler/rustc_mir_transform/src/elaborate_drops.rs @@ -33,16 +33,16 @@ use std::fmt; /// as it would allow running a destructor on a place behind a reference: /// /// ```text -// fn drop_term<T>(t: &mut T) { -// mir! { -// { -// Drop(*t, exit) -// } -// exit = { -// Return() -// } -// } -// } +/// fn drop_term<T>(t: &mut T) { +/// mir! { +/// { +/// Drop(*t, exit) +/// } +/// exit = { +/// Return() +/// } +/// } +/// } /// ``` pub struct ElaborateDrops; diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs index 2b7d9be6d35..1002746e553 100644 --- a/compiler/rustc_mir_transform/src/gvn.rs +++ b/compiler/rustc_mir_transform/src/gvn.rs @@ -1391,11 +1391,15 @@ fn op_to_prop_const<'tcx>( let (prov, offset) = pointer.into_parts(); let alloc_id = prov.alloc_id(); intern_const_alloc_for_constprop(ecx, alloc_id).ok()?; - if matches!(ecx.tcx.global_alloc(alloc_id), GlobalAlloc::Memory(_)) { - // `alloc_id` may point to a static. Codegen will choke on an `Indirect` with anything - // by `GlobalAlloc::Memory`, so do fall through to copying if needed. - // FIXME: find a way to treat this more uniformly - // (probably by fixing codegen) + + // `alloc_id` may point to a static. Codegen will choke on an `Indirect` with anything + // by `GlobalAlloc::Memory`, so do fall through to copying if needed. + // FIXME: find a way to treat this more uniformly (probably by fixing codegen) + if let GlobalAlloc::Memory(alloc) = ecx.tcx.global_alloc(alloc_id) + // Transmuting a constant is just an offset in the allocation. If the alignment of the + // allocation is not enough, fallback to copying into a properly aligned value. + && alloc.inner().align >= op.layout.align.abi + { return Some(ConstValue::Indirect { alloc_id, offset }); } } diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 5075e072754..fd9f0fec88d 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -41,6 +41,12 @@ struct CallSite<'tcx> { impl<'tcx> MirPass<'tcx> for Inline { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + // FIXME(#127234): Coverage instrumentation currently doesn't handle inlined + // MIR correctly when Modified Condition/Decision Coverage is enabled. + if sess.instrument_coverage_mcdc() { + return false; + } + if let Some(enabled) = sess.opts.unstable_opts.inline_mir { return enabled; } @@ -383,6 +389,8 @@ impl<'tcx> Inliner<'tcx> { ) -> Option<CallSite<'tcx>> { // Only consider direct calls to functions let terminator = bb_data.terminator(); + + // FIXME(explicit_tail_calls): figure out if we can inline tail calls if let TerminatorKind::Call { ref func, fn_span, .. } = terminator.kind { let func_ty = func.ty(caller_body, self.tcx); if let ty::FnDef(def_id, args) = *func_ty.kind() { @@ -550,6 +558,9 @@ impl<'tcx> Inliner<'tcx> { // inline-asm is detected. LLVM will still possibly do an inline later on // if the no-attribute function ends up with the same instruction set anyway. return Err("Cannot move inline-asm across instruction sets"); + } else if let TerminatorKind::TailCall { .. } = term.kind { + // FIXME(explicit_tail_calls): figure out how exactly functions containing tail calls can be inlined (and if they even should) + return Err("can't inline functions with tail calls"); } else { work_list.extend(term.successors()) } @@ -1038,6 +1049,10 @@ impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> { *target = self.map_block(*target); *unwind = self.map_unwind(*unwind); } + TerminatorKind::TailCall { .. } => { + // check_mir_body forbids tail calls + unreachable!() + } TerminatorKind::Call { ref mut target, ref mut unwind, .. } => { if let Some(ref mut tgt) = *target { *tgt = self.map_block(*tgt); diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index 85a61f95ca3..97ec0cb39de 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -596,6 +596,7 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { TerminatorKind::UnwindResume | TerminatorKind::UnwindTerminate(_) | TerminatorKind::Return + | TerminatorKind::TailCall { .. } | TerminatorKind::Unreachable | TerminatorKind::CoroutineDrop => bug!("{term:?} has no terminators"), // Disallowed during optimizations. diff --git a/compiler/rustc_mir_transform/src/known_panics_lint.rs b/compiler/rustc_mir_transform/src/known_panics_lint.rs index 8d6c00bbedb..82ad8879d17 100644 --- a/compiler/rustc_mir_transform/src/known_panics_lint.rs +++ b/compiler/rustc_mir_transform/src/known_panics_lint.rs @@ -799,6 +799,7 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> { | TerminatorKind::UnwindResume | TerminatorKind::UnwindTerminate(_) | TerminatorKind::Return + | TerminatorKind::TailCall { .. } | TerminatorKind::Unreachable | TerminatorKind::Drop { .. } | TerminatorKind::Yield { .. } diff --git a/compiler/rustc_mir_transform/src/mentioned_items.rs b/compiler/rustc_mir_transform/src/mentioned_items.rs index db2bb60bdac..d928d7cf764 100644 --- a/compiler/rustc_mir_transform/src/mentioned_items.rs +++ b/compiler/rustc_mir_transform/src/mentioned_items.rs @@ -38,7 +38,7 @@ impl<'tcx> Visitor<'tcx> for MentionedItemsVisitor<'_, 'tcx> { self.super_terminator(terminator, location); let span = || self.body.source_info(location).span; match &terminator.kind { - mir::TerminatorKind::Call { func, .. } => { + mir::TerminatorKind::Call { func, .. } | mir::TerminatorKind::TailCall { func, .. } => { let callee_ty = func.ty(self.body, self.tcx); self.mentioned_items .push(Spanned { node: MentionedItem::Fn(callee_ty), span: span() }); diff --git a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs index fb52bfa468a..1df5737e859 100644 --- a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs +++ b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs @@ -75,6 +75,7 @@ impl RemoveNoopLandingPads { | TerminatorKind::UnwindTerminate(_) | TerminatorKind::Unreachable | TerminatorKind::Call { .. } + | TerminatorKind::TailCall { .. } | TerminatorKind::Assert { .. } | TerminatorKind::Drop { .. } | TerminatorKind::InlineAsm { .. } => false, diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index 2018a8fe667..ab5c25c4937 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -400,40 +400,44 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> { self.check_edge(location, *target, EdgeKind::Normal); self.check_unwind_edge(location, *unwind); } - TerminatorKind::Call { args, destination, target, unwind, .. } => { - if let Some(target) = target { - self.check_edge(location, *target, EdgeKind::Normal); - } - self.check_unwind_edge(location, *unwind); + TerminatorKind::Call { args, .. } | TerminatorKind::TailCall { args, .. } => { + // FIXME(explicit_tail_calls): refactor this & add tail-call specific checks + if let TerminatorKind::Call { target, unwind, destination, .. } = terminator.kind { + if let Some(target) = target { + self.check_edge(location, target, EdgeKind::Normal); + } + self.check_unwind_edge(location, unwind); + + // The code generation assumes that there are no critical call edges. The assumption + // is used to simplify inserting code that should be executed along the return edge + // from the call. FIXME(tmiasko): Since this is a strictly code generation concern, + // the code generation should be responsible for handling it. + if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Optimized) + && self.is_critical_call_edge(target, unwind) + { + self.fail( + location, + format!( + "encountered critical edge in `Call` terminator {:?}", + terminator.kind, + ), + ); + } - // The code generation assumes that there are no critical call edges. The assumption - // is used to simplify inserting code that should be executed along the return edge - // from the call. FIXME(tmiasko): Since this is a strictly code generation concern, - // the code generation should be responsible for handling it. - if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Optimized) - && self.is_critical_call_edge(*target, *unwind) - { - self.fail( - location, - format!( - "encountered critical edge in `Call` terminator {:?}", - terminator.kind, - ), - ); + // The call destination place and Operand::Move place used as an argument might be + // passed by a reference to the callee. Consequently they cannot be packed. + if is_within_packed(self.tcx, &self.body.local_decls, destination).is_some() { + // This is bad! The callee will expect the memory to be aligned. + self.fail( + location, + format!( + "encountered packed place in `Call` terminator destination: {:?}", + terminator.kind, + ), + ); + } } - // The call destination place and Operand::Move place used as an argument might be - // passed by a reference to the callee. Consequently they cannot be packed. - if is_within_packed(self.tcx, &self.body.local_decls, *destination).is_some() { - // This is bad! The callee will expect the memory to be aligned. - self.fail( - location, - format!( - "encountered packed place in `Call` terminator destination: {:?}", - terminator.kind, - ), - ); - } for arg in args { if let Operand::Move(place) = &arg.node { if is_within_packed(self.tcx, &self.body.local_decls, *place).is_some() { @@ -1498,15 +1502,22 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } } - TerminatorKind::Call { func, .. } => { + TerminatorKind::Call { func, .. } | TerminatorKind::TailCall { func, .. } => { let func_ty = func.ty(&self.body.local_decls, self.tcx); match func_ty.kind() { ty::FnPtr(..) | ty::FnDef(..) => {} _ => self.fail( location, - format!("encountered non-callable type {func_ty} in `Call` terminator"), + format!( + "encountered non-callable type {func_ty} in `{}` terminator", + terminator.kind.name() + ), ), } + + if let TerminatorKind::TailCall { .. } = terminator.kind { + // FIXME(explicit_tail_calls): implement tail-call specific checks here (such as signature matching, forbidding closures, etc) + } } TerminatorKind::Assert { cond, .. } => { let cond_ty = cond.ty(&self.body.local_decls, self.tcx); |