diff options
Diffstat (limited to 'compiler/rustc_mir_transform/src')
68 files changed, 672 insertions, 701 deletions
diff --git a/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs b/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs index edb6bc4fbea..e4bc6b3efe4 100644 --- a/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs +++ b/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs @@ -22,7 +22,7 @@ use rustc_target::spec::PanicStrategy; #[derive(PartialEq)] pub struct AbortUnwindingCalls; -impl<'tcx> MirPass<'tcx> for AbortUnwindingCalls { +impl<'tcx> crate::MirPass<'tcx> for AbortUnwindingCalls { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let def_id = body.source.def_id(); let kind = tcx.def_kind(def_id); diff --git a/compiler/rustc_mir_transform/src/add_call_guards.rs b/compiler/rustc_mir_transform/src/add_call_guards.rs index a47c8d94bba..78e850de3c7 100644 --- a/compiler/rustc_mir_transform/src/add_call_guards.rs +++ b/compiler/rustc_mir_transform/src/add_call_guards.rs @@ -1,6 +1,7 @@ use rustc_index::{Idx, IndexVec}; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; +use tracing::debug; #[derive(PartialEq)] pub enum AddCallGuards { @@ -29,7 +30,7 @@ pub use self::AddCallGuards::*; * */ -impl<'tcx> MirPass<'tcx> for AddCallGuards { +impl<'tcx> crate::MirPass<'tcx> for AddCallGuards { fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { self.add_call_guards(body); } diff --git a/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs b/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs index cd850e2d731..4a8196aeff5 100644 --- a/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs +++ b/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs @@ -1,6 +1,7 @@ use rustc_middle::mir::patch::MirPatch; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; +use tracing::debug; use crate::util; @@ -36,7 +37,7 @@ use crate::util; /// blowup. pub struct AddMovesForPackedDrops; -impl<'tcx> MirPass<'tcx> for AddMovesForPackedDrops { +impl<'tcx> crate::MirPass<'tcx> for AddMovesForPackedDrops { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { debug!("add_moves_for_packed_drops({:?} @ {:?})", body.source, body.span); add_moves_for_packed_drops(tcx, body); @@ -85,7 +86,7 @@ fn add_move_for_packed_drop<'tcx>( let source_info = terminator.source_info; let ty = place.ty(body, tcx).ty; - let temp = patch.new_temp(ty, terminator.source_info.span); + let temp = patch.new_temp(ty, source_info.span); let storage_dead_block = patch.new_block(BasicBlockData { statements: vec![Statement { source_info, kind: StatementKind::StorageDead(temp) }], diff --git a/compiler/rustc_mir_transform/src/add_retag.rs b/compiler/rustc_mir_transform/src/add_retag.rs index 12a68790374..2e12064fe73 100644 --- a/compiler/rustc_mir_transform/src/add_retag.rs +++ b/compiler/rustc_mir_transform/src/add_retag.rs @@ -48,7 +48,7 @@ fn may_contain_reference<'tcx>(ty: Ty<'tcx>, depth: u32, tcx: TyCtxt<'tcx>) -> b } } -impl<'tcx> MirPass<'tcx> for AddRetag { +impl<'tcx> crate::MirPass<'tcx> for AddRetag { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { sess.opts.unstable_opts.mir_emit_retag } @@ -137,6 +137,8 @@ impl<'tcx> MirPass<'tcx> for AddRetag { // Using `is_box_global` here is a bit sketchy: if this code is // generic over the allocator, we'll not add a retag! This is a hack // to make Stacked Borrows compatible with custom allocator code. + // It means the raw pointer inherits the tag of the box, which mostly works + // but can sometimes lead to unexpected aliasing errors. // Long-term, we'll want to move to an aliasing model where "cast to // raw pointer" is a complete NOP, and then this will no longer be // an issue. diff --git a/compiler/rustc_mir_transform/src/add_subtyping_projections.rs b/compiler/rustc_mir_transform/src/add_subtyping_projections.rs index 04204c68f7b..369f6c60084 100644 --- a/compiler/rustc_mir_transform/src/add_subtyping_projections.rs +++ b/compiler/rustc_mir_transform/src/add_subtyping_projections.rs @@ -62,7 +62,7 @@ pub fn subtype_finder<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { checker.patcher.apply(body); } -impl<'tcx> MirPass<'tcx> for Subtyper { +impl<'tcx> crate::MirPass<'tcx> for Subtyper { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { subtype_finder(tcx, body); } diff --git a/compiler/rustc_mir_transform/src/check_alignment.rs b/compiler/rustc_mir_transform/src/check_alignment.rs index 5dfdcfc8b94..2e072aa262a 100644 --- a/compiler/rustc_mir_transform/src/check_alignment.rs +++ b/compiler/rustc_mir_transform/src/check_alignment.rs @@ -5,10 +5,11 @@ use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceC use rustc_middle::mir::*; use rustc_middle::ty::{self, ParamEnv, Ty, TyCtxt}; use rustc_session::Session; +use tracing::{debug, trace}; pub struct CheckAlignment; -impl<'tcx> MirPass<'tcx> for CheckAlignment { +impl<'tcx> crate::MirPass<'tcx> for CheckAlignment { fn is_enabled(&self, sess: &Session) -> bool { // FIXME(#112480) MSVC and rustc disagree on minimum stack alignment on x86 Windows if sess.target.llvm_target == "i686-pc-windows-msvc" { diff --git a/compiler/rustc_mir_transform/src/check_const_item_mutation.rs b/compiler/rustc_mir_transform/src/check_const_item_mutation.rs index 1f615c9d8d1..fb03323e37e 100644 --- a/compiler/rustc_mir_transform/src/check_const_item_mutation.rs +++ b/compiler/rustc_mir_transform/src/check_const_item_mutation.rs @@ -6,11 +6,11 @@ use rustc_session::lint::builtin::CONST_ITEM_MUTATION; use rustc_span::def_id::DefId; use rustc_span::Span; -use crate::{errors, MirLint}; +use crate::errors; pub struct CheckConstItemMutation; -impl<'tcx> MirLint<'tcx> for CheckConstItemMutation { +impl<'tcx> crate::MirLint<'tcx> for CheckConstItemMutation { fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) { let mut checker = ConstMutationChecker { body, tcx, target_local: None }; checker.visit_body(body); diff --git a/compiler/rustc_mir_transform/src/check_packed_ref.rs b/compiler/rustc_mir_transform/src/check_packed_ref.rs index 9902002580a..2f957de7e78 100644 --- a/compiler/rustc_mir_transform/src/check_packed_ref.rs +++ b/compiler/rustc_mir_transform/src/check_packed_ref.rs @@ -3,11 +3,11 @@ use rustc_middle::mir::*; use rustc_middle::span_bug; use rustc_middle::ty::{self, TyCtxt}; -use crate::{errors, util, MirLint}; +use crate::{errors, util}; pub struct CheckPackedRef; -impl<'tcx> MirLint<'tcx> for CheckPackedRef { +impl<'tcx> crate::MirLint<'tcx> for CheckPackedRef { fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) { let param_env = tcx.param_env(body.source.def_id()); let source_info = SourceInfo::outermost(body.span); @@ -37,24 +37,17 @@ impl<'tcx> Visitor<'tcx> for PackedRefChecker<'_, 'tcx> { } fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, _location: Location) { - if context.is_borrow() { - if util::is_disaligned(self.tcx, self.body, self.param_env, *place) { - let def_id = self.body.source.instance.def_id(); - if let Some(impl_def_id) = self.tcx.impl_of_method(def_id) - && self.tcx.is_builtin_derived(impl_def_id) - { - // If we ever reach here it means that the generated derive - // code is somehow doing an unaligned reference, which it - // shouldn't do. - span_bug!( - self.source_info.span, - "builtin derive created an unaligned reference" - ); - } else { - self.tcx - .dcx() - .emit_err(errors::UnalignedPackedRef { span: self.source_info.span }); - } + if context.is_borrow() && util::is_disaligned(self.tcx, self.body, self.param_env, *place) { + let def_id = self.body.source.instance.def_id(); + if let Some(impl_def_id) = self.tcx.impl_of_method(def_id) + && self.tcx.is_builtin_derived(impl_def_id) + { + // If we ever reach here it means that the generated derive + // code is somehow doing an unaligned reference, which it + // shouldn't do. + span_bug!(self.source_info.span, "builtin derive created an unaligned reference"); + } else { + self.tcx.dcx().emit_err(errors::UnalignedPackedRef { span: self.source_info.span }); } } } diff --git a/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs b/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs index 08c9f9f08e6..2f3be1e425d 100644 --- a/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs +++ b/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs @@ -21,11 +21,9 @@ use rustc_middle::mir::{Body, BorrowKind, CastKind, Rvalue, StatementKind, Termi use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::TyCtxt; -use crate::MirPass; - pub struct CleanupPostBorrowck; -impl<'tcx> MirPass<'tcx> for CleanupPostBorrowck { +impl<'tcx> crate::MirPass<'tcx> for CleanupPostBorrowck { fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { for basic_block in body.basic_blocks.as_mut() { for statement in basic_block.statements.iter_mut() { diff --git a/compiler/rustc_mir_transform/src/copy_prop.rs b/compiler/rustc_mir_transform/src/copy_prop.rs index c1f9313a377..85d25ca2231 100644 --- a/compiler/rustc_mir_transform/src/copy_prop.rs +++ b/compiler/rustc_mir_transform/src/copy_prop.rs @@ -3,6 +3,7 @@ use rustc_index::IndexSlice; use rustc_middle::mir::visit::*; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; +use tracing::{debug, instrument}; use crate::ssa::SsaLocals; @@ -18,7 +19,7 @@ use crate::ssa::SsaLocals; /// We want to replace all those locals by `_a`, either copied or moved. pub struct CopyProp; -impl<'tcx> MirPass<'tcx> for CopyProp { +impl<'tcx> crate::MirPass<'tcx> for CopyProp { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { sess.mir_opt_level() >= 1 } diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs index 82528109be9..eefb748e49d 100644 --- a/compiler/rustc_mir_transform/src/coroutine.rs +++ b/compiler/rustc_mir_transform/src/coroutine.rs @@ -53,7 +53,7 @@ mod by_move_body; use std::{iter, ops}; -pub use by_move_body::ByMoveBody; +pub use by_move_body::coroutine_by_move_body_def_id; use rustc_data_structures::fx::FxHashSet; use rustc_errors::pluralize; use rustc_hir as hir; @@ -63,7 +63,9 @@ use rustc_index::bit_set::{BitMatrix, BitSet, GrowableBitSet}; use rustc_index::{Idx, IndexVec}; use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor}; use rustc_middle::mir::*; -use rustc_middle::ty::{self, CoroutineArgs, CoroutineArgsExt, InstanceKind, Ty, TyCtxt}; +use rustc_middle::ty::{ + self, CoroutineArgs, CoroutineArgsExt, GenericArgsRef, InstanceKind, Ty, TyCtxt, +}; use rustc_middle::{bug, span_bug}; use rustc_mir_dataflow::impls::{ MaybeBorrowedLocals, MaybeLiveLocals, MaybeRequiresStorage, MaybeStorageLive, @@ -78,6 +80,7 @@ use rustc_target::spec::PanicStrategy; use rustc_trait_selection::error_reporting::InferCtxtErrorExt; use rustc_trait_selection::infer::TyCtxtInferExt as _; use rustc_trait_selection::traits::{ObligationCause, ObligationCauseCode, ObligationCtxt}; +use tracing::{debug, instrument, trace}; use crate::deref_separator::deref_finder; use crate::{abort_unwinding_calls, errors, pass_manager as pm, simplify}; @@ -112,47 +115,18 @@ impl<'tcx> MutVisitor<'tcx> for RenameLocalVisitor<'tcx> { } } -struct DerefArgVisitor<'tcx> { +struct SelfArgVisitor<'tcx> { tcx: TyCtxt<'tcx>, + new_base: Place<'tcx>, } -impl<'tcx> MutVisitor<'tcx> for DerefArgVisitor<'tcx> { - fn tcx(&self) -> TyCtxt<'tcx> { - self.tcx - } - - fn visit_local(&mut self, local: &mut Local, _: PlaceContext, _: Location) { - assert_ne!(*local, SELF_ARG); - } - - fn visit_place(&mut self, place: &mut Place<'tcx>, context: PlaceContext, location: Location) { - if place.local == SELF_ARG { - replace_base( - place, - Place { - local: SELF_ARG, - projection: self.tcx().mk_place_elems(&[ProjectionElem::Deref]), - }, - self.tcx, - ); - } else { - self.visit_local(&mut place.local, context, location); - - for elem in place.projection.iter() { - if let PlaceElem::Index(local) = elem { - assert_ne!(local, SELF_ARG); - } - } - } +impl<'tcx> SelfArgVisitor<'tcx> { + fn new(tcx: TyCtxt<'tcx>, elem: ProjectionElem<Local, Ty<'tcx>>) -> Self { + Self { tcx, new_base: Place { local: SELF_ARG, projection: tcx.mk_place_elems(&[elem]) } } } } -struct PinArgVisitor<'tcx> { - ref_coroutine_ty: Ty<'tcx>, - tcx: TyCtxt<'tcx>, -} - -impl<'tcx> MutVisitor<'tcx> for PinArgVisitor<'tcx> { +impl<'tcx> MutVisitor<'tcx> for SelfArgVisitor<'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } @@ -163,17 +137,7 @@ impl<'tcx> MutVisitor<'tcx> for PinArgVisitor<'tcx> { fn visit_place(&mut self, place: &mut Place<'tcx>, context: PlaceContext, location: Location) { if place.local == SELF_ARG { - replace_base( - place, - Place { - local: SELF_ARG, - projection: self.tcx().mk_place_elems(&[ProjectionElem::Field( - FieldIdx::ZERO, - self.ref_coroutine_ty, - )]), - }, - self.tcx, - ); + replace_base(place, self.new_base, self.tcx); } else { self.visit_local(&mut place.local, context, location); @@ -197,15 +161,6 @@ fn replace_base<'tcx>(place: &mut Place<'tcx>, new_base: Place<'tcx>, tcx: TyCtx const SELF_ARG: Local = Local::from_u32(1); -/// Coroutine has not been resumed yet. -const UNRESUMED: usize = CoroutineArgs::UNRESUMED; -/// Coroutine has returned / is completed. -const RETURNED: usize = CoroutineArgs::RETURNED; -/// Coroutine has panicked and is poisoned. -const POISONED: usize = CoroutineArgs::POISONED; -/// Number of reserved variants of coroutine state. -const RESERVED_VARIANTS: usize = CoroutineArgs::RESERVED_VARIANTS; - /// A `yield` point in the coroutine. struct SuspensionPoint<'tcx> { /// State discriminant used when suspending or resuming at this point. @@ -260,14 +215,10 @@ impl<'tcx> TransformVisitor<'tcx> { // `gen` continues return `None` CoroutineKind::Desugared(CoroutineDesugaring::Gen, _) => { let option_def_id = self.tcx.require_lang_item(LangItem::Option, None); - Rvalue::Aggregate( - Box::new(AggregateKind::Adt( - option_def_id, - VariantIdx::ZERO, - self.tcx.mk_args(&[self.old_yield_ty.into()]), - None, - None, - )), + make_aggregate_adt( + option_def_id, + VariantIdx::ZERO, + self.tcx.mk_args(&[self.old_yield_ty.into()]), IndexVec::new(), ) } @@ -316,64 +267,28 @@ impl<'tcx> TransformVisitor<'tcx> { is_return: bool, statements: &mut Vec<Statement<'tcx>>, ) { + const ZERO: VariantIdx = VariantIdx::ZERO; + const ONE: VariantIdx = VariantIdx::from_usize(1); let rvalue = match self.coroutine_kind { CoroutineKind::Desugared(CoroutineDesugaring::Async, _) => { let poll_def_id = self.tcx.require_lang_item(LangItem::Poll, None); let args = self.tcx.mk_args(&[self.old_ret_ty.into()]); - if is_return { - // Poll::Ready(val) - Rvalue::Aggregate( - Box::new(AggregateKind::Adt( - poll_def_id, - VariantIdx::ZERO, - args, - None, - None, - )), - IndexVec::from_raw(vec![val]), - ) + let (variant_idx, operands) = if is_return { + (ZERO, IndexVec::from_raw(vec![val])) // Poll::Ready(val) } else { - // Poll::Pending - Rvalue::Aggregate( - Box::new(AggregateKind::Adt( - poll_def_id, - VariantIdx::from_usize(1), - args, - None, - None, - )), - IndexVec::new(), - ) - } + (ONE, IndexVec::new()) // Poll::Pending + }; + make_aggregate_adt(poll_def_id, variant_idx, args, operands) } CoroutineKind::Desugared(CoroutineDesugaring::Gen, _) => { let option_def_id = self.tcx.require_lang_item(LangItem::Option, None); let args = self.tcx.mk_args(&[self.old_yield_ty.into()]); - if is_return { - // None - Rvalue::Aggregate( - Box::new(AggregateKind::Adt( - option_def_id, - VariantIdx::ZERO, - args, - None, - None, - )), - IndexVec::new(), - ) + let (variant_idx, operands) = if is_return { + (ZERO, IndexVec::new()) // None } else { - // Some(val) - Rvalue::Aggregate( - Box::new(AggregateKind::Adt( - option_def_id, - VariantIdx::from_usize(1), - args, - None, - None, - )), - IndexVec::from_raw(vec![val]), - ) - } + (ONE, IndexVec::from_raw(vec![val])) // Some(val) + }; + make_aggregate_adt(option_def_id, variant_idx, args, operands) } CoroutineKind::Desugared(CoroutineDesugaring::AsyncGen, _) => { if is_return { @@ -399,31 +314,17 @@ impl<'tcx> TransformVisitor<'tcx> { let coroutine_state_def_id = self.tcx.require_lang_item(LangItem::CoroutineState, None); let args = self.tcx.mk_args(&[self.old_yield_ty.into(), self.old_ret_ty.into()]); - if is_return { - // CoroutineState::Complete(val) - Rvalue::Aggregate( - Box::new(AggregateKind::Adt( - coroutine_state_def_id, - VariantIdx::from_usize(1), - args, - None, - None, - )), - IndexVec::from_raw(vec![val]), - ) + let variant_idx = if is_return { + ONE // CoroutineState::Complete(val) } else { - // CoroutineState::Yielded(val) - Rvalue::Aggregate( - Box::new(AggregateKind::Adt( - coroutine_state_def_id, - VariantIdx::ZERO, - args, - None, - None, - )), - IndexVec::from_raw(vec![val]), - ) - } + ZERO // CoroutineState::Yielded(val) + }; + make_aggregate_adt( + coroutine_state_def_id, + variant_idx, + args, + IndexVec::from_raw(vec![val]), + ) } }; @@ -516,7 +417,7 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { self.make_state(v, source_info, is_return, &mut data.statements); let state = if let Some((resume, mut resume_arg)) = resume { // Yield - let state = RESERVED_VARIANTS + self.suspension_points.len(); + let state = CoroutineArgs::RESERVED_VARIANTS + self.suspension_points.len(); // The resume arg target location might itself be remapped if its base local is // live across a yield. @@ -549,7 +450,7 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { VariantIdx::new(state) } else { // Return - VariantIdx::new(RETURNED) // state for returned + VariantIdx::new(CoroutineArgs::RETURNED) // state for returned }; data.statements.push(self.set_discr(state, source_info)); data.terminator_mut().kind = TerminatorKind::Return; @@ -559,6 +460,15 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { } } +fn make_aggregate_adt<'tcx>( + def_id: DefId, + variant_idx: VariantIdx, + args: GenericArgsRef<'tcx>, + operands: IndexVec<FieldIdx, Operand<'tcx>>, +) -> Rvalue<'tcx> { + Rvalue::Aggregate(Box::new(AggregateKind::Adt(def_id, variant_idx, args, None, None)), operands) +} + fn make_coroutine_state_argument_indirect<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let coroutine_ty = body.local_decls.raw[1].ty; @@ -568,7 +478,7 @@ fn make_coroutine_state_argument_indirect<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Bo body.local_decls.raw[1].ty = ref_coroutine_ty; // Add a deref to accesses of the coroutine state - DerefArgVisitor { tcx }.visit_body(body); + SelfArgVisitor::new(tcx, ProjectionElem::Deref).visit_body(body); } fn make_coroutine_state_argument_pinned<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { @@ -583,7 +493,8 @@ fn make_coroutine_state_argument_pinned<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body body.local_decls.raw[1].ty = pin_ref_coroutine_ty; // Add the Pin field access to accesses of the coroutine state - PinArgVisitor { ref_coroutine_ty, tcx }.visit_body(body); + SelfArgVisitor::new(tcx, ProjectionElem::Field(FieldIdx::ZERO, ref_coroutine_ty)) + .visit_body(body); } /// Allocates a new local and replaces all references of `local` with it. Returns the new local. @@ -650,8 +561,6 @@ fn transform_async_context<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let local = eliminate_get_context_call(&mut body[bb]); replace_resume_ty_local(tcx, body, local, context_mut_ref); } - } else { - continue; } } TerminatorKind::Yield { resume_arg, .. } => { @@ -664,24 +573,23 @@ fn transform_async_context<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { fn eliminate_get_context_call<'tcx>(bb_data: &mut BasicBlockData<'tcx>) -> Local { let terminator = bb_data.terminator.take().unwrap(); - if let TerminatorKind::Call { args, destination, target, .. } = terminator.kind { - let [arg] = *Box::try_from(args).unwrap(); - let local = arg.node.place().unwrap().local; - - let arg = Rvalue::Use(arg.node); - let assign = Statement { - source_info: terminator.source_info, - kind: StatementKind::Assign(Box::new((destination, arg))), - }; - bb_data.statements.push(assign); - bb_data.terminator = Some(Terminator { - source_info: terminator.source_info, - kind: TerminatorKind::Goto { target: target.unwrap() }, - }); - local - } else { + let TerminatorKind::Call { args, destination, target, .. } = terminator.kind else { bug!(); - } + }; + let [arg] = *Box::try_from(args).unwrap(); + let local = arg.node.place().unwrap().local; + + let arg = Rvalue::Use(arg.node); + let assign = Statement { + source_info: terminator.source_info, + kind: StatementKind::Assign(Box::new((destination, arg))), + }; + bb_data.statements.push(assign); + bb_data.terminator = Some(Terminator { + source_info: terminator.source_info, + kind: TerminatorKind::Goto { target: target.unwrap() }, + }); + local } #[cfg_attr(not(debug_assertions), allow(unused))] @@ -931,7 +839,7 @@ fn compute_storage_conflicts<'mir, 'tcx>( // Compute the storage conflicts for all eligible locals. let mut visitor = StorageConflictVisitor { body, - saved_locals: saved_locals, + saved_locals, local_conflicts: BitMatrix::from_row_n(&ineligible_locals, body.local_decls.len()), eligible_storage_live: BitSet::new_empty(body.local_decls.len()), }; @@ -1084,10 +992,11 @@ fn compute_layout<'tcx>( // Build the coroutine variant field list. // Create a map from local indices to coroutine struct indices. let mut variant_fields: IndexVec<VariantIdx, IndexVec<FieldIdx, CoroutineSavedLocal>> = - iter::repeat(IndexVec::new()).take(RESERVED_VARIANTS).collect(); + iter::repeat(IndexVec::new()).take(CoroutineArgs::RESERVED_VARIANTS).collect(); let mut remap = IndexVec::from_elem_n(None, saved_locals.domain_size()); for (suspension_point_idx, live_locals) in live_locals_at_suspension_points.iter().enumerate() { - let variant_index = VariantIdx::from(RESERVED_VARIANTS + suspension_point_idx); + let variant_index = + VariantIdx::from(CoroutineArgs::RESERVED_VARIANTS + suspension_point_idx); let mut fields = IndexVec::new(); for (idx, saved_local) in live_locals.iter().enumerate() { fields.push(saved_local); @@ -1182,12 +1091,10 @@ fn elaborate_coroutine_drops<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { source_info, kind: TerminatorKind::Drop { place, target, unwind, replace: _ }, } => { - if let Some(local) = place.as_local() { - if local == SELF_ARG { - (target, unwind, source_info) - } else { - continue; - } + if let Some(local) = place.as_local() + && local == SELF_ARG + { + (target, unwind, source_info) } else { continue; } @@ -1236,7 +1143,7 @@ fn create_coroutine_drop_shim<'tcx>( let mut cases = create_cases(&mut body, transform, Operation::Drop); - cases.insert(0, (UNRESUMED, drop_clean)); + cases.insert(0, (CoroutineArgs::UNRESUMED, drop_clean)); // The returned state and the poisoned state fall through to the default // case which is just to return @@ -1386,7 +1293,9 @@ fn create_coroutine_resume_function<'tcx>( if can_unwind { let source_info = SourceInfo::outermost(body.span); let poison_block = body.basic_blocks_mut().push(BasicBlockData { - statements: vec![transform.set_discr(VariantIdx::new(POISONED), source_info)], + statements: vec![ + transform.set_discr(VariantIdx::new(CoroutineArgs::POISONED), source_info), + ], terminator: Some(Terminator { source_info, kind: TerminatorKind::UnwindResume }), is_cleanup: true, }); @@ -1418,13 +1327,16 @@ fn create_coroutine_resume_function<'tcx>( use rustc_middle::mir::AssertKind::{ResumedAfterPanic, ResumedAfterReturn}; // Jump to the entry point on the unresumed - cases.insert(0, (UNRESUMED, START_BLOCK)); + cases.insert(0, (CoroutineArgs::UNRESUMED, START_BLOCK)); // Panic when resumed on the returned or poisoned state if can_unwind { cases.insert( 1, - (POISONED, insert_panic_block(tcx, body, ResumedAfterPanic(transform.coroutine_kind))), + ( + CoroutineArgs::POISONED, + insert_panic_block(tcx, body, ResumedAfterPanic(transform.coroutine_kind)), + ), ); } @@ -1439,7 +1351,7 @@ fn create_coroutine_resume_function<'tcx>( transform.insert_none_ret_block(body) } }; - cases.insert(1, (RETURNED, block)); + cases.insert(1, (CoroutineArgs::RETURNED, block)); } insert_switch(body, cases, &transform, TerminatorKind::Unreachable); @@ -1623,7 +1535,7 @@ fn check_field_tys_sized<'tcx>( } } -impl<'tcx> MirPass<'tcx> for StateTransform { +impl<'tcx> crate::MirPass<'tcx> for StateTransform { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let Some(old_yield_ty) = body.yield_ty() else { // This only applies to coroutines diff --git a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs index 7ea36f08232..cf39c136b01 100644 --- a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs +++ b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs @@ -69,161 +69,168 @@ //! in case precise captures (edition 2021 closure capture rules) caused the inner coroutine //! to split one field capture into two. +use rustc_data_structures::steal::Steal; use rustc_data_structures::unord::UnordMap; use rustc_hir as hir; +use rustc_hir::def::DefKind; +use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_middle::bug; use rustc_middle::hir::place::{Projection, ProjectionKind}; use rustc_middle::mir::visit::MutVisitor; -use rustc_middle::mir::{self, dump_mir, MirPass}; +use rustc_middle::mir::{self, dump_mir}; use rustc_middle::ty::{self, InstanceKind, Ty, TyCtxt, TypeVisitableExt}; +use rustc_span::symbol::kw; use rustc_target::abi::{FieldIdx, VariantIdx}; -use crate::pass_manager::validate_body; - -pub struct ByMoveBody; +pub fn coroutine_by_move_body_def_id<'tcx>( + tcx: TyCtxt<'tcx>, + coroutine_def_id: LocalDefId, +) -> DefId { + let body = tcx.mir_built(coroutine_def_id).borrow(); -impl<'tcx> MirPass<'tcx> for ByMoveBody { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut mir::Body<'tcx>) { - // We only need to generate by-move coroutine bodies for coroutines that come - // from coroutine-closures. - let Some(coroutine_def_id) = body.source.def_id().as_local() else { - return; - }; - let Some(hir::CoroutineKind::Desugared(_, hir::CoroutineSource::Closure)) = - tcx.coroutine_kind(coroutine_def_id) - else { - return; - }; + let Some(hir::CoroutineKind::Desugared(_, hir::CoroutineSource::Closure)) = + tcx.coroutine_kind(coroutine_def_id) + else { + bug!("should only be invoked on coroutine-closures"); + }; - // Also, let's skip processing any bodies with errors, since there's no guarantee - // the MIR body will be constructed well. - let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; - if coroutine_ty.references_error() { - return; - } + // Also, let's skip processing any bodies with errors, since there's no guarantee + // the MIR body will be constructed well. + let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; - // We don't need to generate a by-move coroutine if the coroutine body was - // produced by the `CoroutineKindShim`, since it's already by-move. - if matches!(body.source.instance, ty::InstanceKind::CoroutineKindShim { .. }) { - return; - } + let ty::Coroutine(_, args) = *coroutine_ty.kind() else { bug!("{body:#?}") }; + let args = args.as_coroutine(); - let ty::Coroutine(_, args) = *coroutine_ty.kind() else { bug!("{body:#?}") }; - let args = args.as_coroutine(); + let coroutine_kind = args.kind_ty().to_opt_closure_kind().unwrap(); - let coroutine_kind = args.kind_ty().to_opt_closure_kind().unwrap(); + let parent_def_id = tcx.local_parent(coroutine_def_id); + let ty::CoroutineClosure(_, parent_args) = + *tcx.type_of(parent_def_id).instantiate_identity().kind() + else { + bug!(); + }; + if parent_args.references_error() { + return coroutine_def_id.to_def_id(); + } - let parent_def_id = tcx.local_parent(coroutine_def_id); - let ty::CoroutineClosure(_, parent_args) = - *tcx.type_of(parent_def_id).instantiate_identity().kind() - else { - bug!(); - }; - let parent_closure_args = parent_args.as_coroutine_closure(); - let num_args = parent_closure_args - .coroutine_closure_sig() - .skip_binder() - .tupled_inputs_ty - .tuple_fields() - .len(); + let parent_closure_args = parent_args.as_coroutine_closure(); + let num_args = parent_closure_args + .coroutine_closure_sig() + .skip_binder() + .tupled_inputs_ty + .tuple_fields() + .len(); - let field_remapping: UnordMap<_, _> = ty::analyze_coroutine_closure_captures( - tcx.closure_captures(parent_def_id).iter().copied(), - tcx.closure_captures(coroutine_def_id).iter().skip(num_args).copied(), - |(parent_field_idx, parent_capture), (child_field_idx, child_capture)| { - // Store this set of additional projections (fields and derefs). - // We need to re-apply them later. - let mut child_precise_captures = child_capture.place.projections - [parent_capture.place.projections.len()..] - .to_vec(); + let field_remapping: UnordMap<_, _> = ty::analyze_coroutine_closure_captures( + tcx.closure_captures(parent_def_id).iter().copied(), + tcx.closure_captures(coroutine_def_id).iter().skip(num_args).copied(), + |(parent_field_idx, parent_capture), (child_field_idx, child_capture)| { + // Store this set of additional projections (fields and derefs). + // We need to re-apply them later. + let mut child_precise_captures = + child_capture.place.projections[parent_capture.place.projections.len()..].to_vec(); - // If the parent capture is by-ref, then we need to apply an additional - // deref before applying any further projections to this place. - if parent_capture.is_by_ref() { - child_precise_captures.insert( - 0, - Projection { ty: parent_capture.place.ty(), kind: ProjectionKind::Deref }, - ); - } - // If the child capture is by-ref, then we need to apply a "ref" - // projection (i.e. `&`) at the end. But wait! We don't have that - // as a projection kind. So instead, we can apply its dual and - // *peel* a deref off of the place when it shows up in the MIR body. - // Luckily, by construction this is always possible. - let peel_deref = if child_capture.is_by_ref() { - assert!( - parent_capture.is_by_ref() || coroutine_kind != ty::ClosureKind::FnOnce, - "`FnOnce` coroutine-closures return coroutines that capture from \ + // If the parent capture is by-ref, then we need to apply an additional + // deref before applying any further projections to this place. + if parent_capture.is_by_ref() { + child_precise_captures.insert( + 0, + Projection { ty: parent_capture.place.ty(), kind: ProjectionKind::Deref }, + ); + } + // If the child capture is by-ref, then we need to apply a "ref" + // projection (i.e. `&`) at the end. But wait! We don't have that + // as a projection kind. So instead, we can apply its dual and + // *peel* a deref off of the place when it shows up in the MIR body. + // Luckily, by construction this is always possible. + let peel_deref = if child_capture.is_by_ref() { + assert!( + parent_capture.is_by_ref() || coroutine_kind != ty::ClosureKind::FnOnce, + "`FnOnce` coroutine-closures return coroutines that capture from \ their body; it will always result in a borrowck error!" - ); - true - } else { - false - }; + ); + true + } else { + false + }; - // Regarding the behavior above, you may think that it's redundant to both - // insert a deref and then peel a deref if the parent and child are both - // captured by-ref. This would be correct, except for the case where we have - // precise capturing projections, since the inserted deref is to the *beginning* - // and the peeled deref is at the *end*. I cannot seem to actually find a - // case where this happens, though, but let's keep this code flexible. + // Regarding the behavior above, you may think that it's redundant to both + // insert a deref and then peel a deref if the parent and child are both + // captured by-ref. This would be correct, except for the case where we have + // precise capturing projections, since the inserted deref is to the *beginning* + // and the peeled deref is at the *end*. I cannot seem to actually find a + // case where this happens, though, but let's keep this code flexible. - // Finally, store the type of the parent's captured place. We need - // this when building the field projection in the MIR body later on. - let mut parent_capture_ty = parent_capture.place.ty(); - parent_capture_ty = match parent_capture.info.capture_kind { - ty::UpvarCapture::ByValue => parent_capture_ty, - ty::UpvarCapture::ByRef(kind) => Ty::new_ref( - tcx, - tcx.lifetimes.re_erased, - parent_capture_ty, - kind.to_mutbl_lossy(), - ), - }; + // Finally, store the type of the parent's captured place. We need + // this when building the field projection in the MIR body later on. + let mut parent_capture_ty = parent_capture.place.ty(); + parent_capture_ty = match parent_capture.info.capture_kind { + ty::UpvarCapture::ByValue => parent_capture_ty, + ty::UpvarCapture::ByRef(kind) => Ty::new_ref( + tcx, + tcx.lifetimes.re_erased, + parent_capture_ty, + kind.to_mutbl_lossy(), + ), + }; + ( + FieldIdx::from_usize(child_field_idx + num_args), ( - FieldIdx::from_usize(child_field_idx + num_args), - ( - FieldIdx::from_usize(parent_field_idx + num_args), - parent_capture_ty, - peel_deref, - child_precise_captures, - ), - ) - }, - ) - .collect(); + FieldIdx::from_usize(parent_field_idx + num_args), + parent_capture_ty, + peel_deref, + child_precise_captures, + ), + ) + }, + ) + .collect(); - if coroutine_kind == ty::ClosureKind::FnOnce { - assert_eq!(field_remapping.len(), tcx.closure_captures(parent_def_id).len()); - return; - } + if coroutine_kind == ty::ClosureKind::FnOnce { + assert_eq!(field_remapping.len(), tcx.closure_captures(parent_def_id).len()); + // The by-move body is just the body :) + return coroutine_def_id.to_def_id(); + } - let by_move_coroutine_ty = tcx - .instantiate_bound_regions_with_erased(parent_closure_args.coroutine_closure_sig()) - .to_coroutine_given_kind_and_upvars( - tcx, - parent_closure_args.parent_args(), - coroutine_def_id.to_def_id(), - ty::ClosureKind::FnOnce, - tcx.lifetimes.re_erased, - parent_closure_args.tupled_upvars_ty(), - parent_closure_args.coroutine_captures_by_ref_ty(), - ); + let by_move_coroutine_ty = tcx + .instantiate_bound_regions_with_erased(parent_closure_args.coroutine_closure_sig()) + .to_coroutine_given_kind_and_upvars( + tcx, + parent_closure_args.parent_args(), + coroutine_def_id.to_def_id(), + ty::ClosureKind::FnOnce, + tcx.lifetimes.re_erased, + parent_closure_args.tupled_upvars_ty(), + parent_closure_args.coroutine_captures_by_ref_ty(), + ); - let mut by_move_body = body.clone(); - MakeByMoveBody { tcx, field_remapping, by_move_coroutine_ty }.visit_body(&mut by_move_body); - dump_mir(tcx, false, "coroutine_by_move", &0, &by_move_body, |_, _| Ok(())); + let mut by_move_body = body.clone(); + MakeByMoveBody { tcx, field_remapping, by_move_coroutine_ty }.visit_body(&mut by_move_body); - // Let's just always validate this body. - validate_body(tcx, &mut by_move_body, "Initial coroutine_by_move body".to_string()); + // This will always be `{closure#1}`, since the original coroutine is `{closure#0}`. + let body_def = tcx.create_def(parent_def_id, kw::Empty, DefKind::SyntheticCoroutineBody); + by_move_body.source = + mir::MirSource::from_instance(InstanceKind::Item(body_def.def_id().to_def_id())); + dump_mir(tcx, false, "built", &"after", &by_move_body, |_, _| Ok(())); - // FIXME: use query feeding to generate the body right here and then only store the `DefId` of the new body. - by_move_body.source = mir::MirSource::from_instance(InstanceKind::CoroutineKindShim { - coroutine_def_id: coroutine_def_id.to_def_id(), - }); - body.coroutine.as_mut().unwrap().by_move_body = Some(by_move_body); - } + // Inherited from the by-ref coroutine. + body_def.codegen_fn_attrs(tcx.codegen_fn_attrs(coroutine_def_id).clone()); + body_def.constness(tcx.constness(coroutine_def_id).clone()); + body_def.coroutine_kind(tcx.coroutine_kind(coroutine_def_id).clone()); + body_def.def_ident_span(tcx.def_ident_span(coroutine_def_id)); + body_def.def_span(tcx.def_span(coroutine_def_id)); + body_def.explicit_predicates_of(tcx.explicit_predicates_of(coroutine_def_id).clone()); + body_def.generics_of(tcx.generics_of(coroutine_def_id).clone()); + body_def.param_env(tcx.param_env(coroutine_def_id).clone()); + body_def.predicates_of(tcx.predicates_of(coroutine_def_id).clone()); + + // The type of the coroutine is the `by_move_coroutine_ty`. + body_def.type_of(ty::EarlyBinder::bind(by_move_coroutine_ty)); + + body_def.mir_built(tcx.arena.alloc(Steal::new(by_move_body))); + + body_def.def_id().to_def_id() } struct MakeByMoveBody<'tcx> { diff --git a/compiler/rustc_mir_transform/src/coverage/counters.rs b/compiler/rustc_mir_transform/src/coverage/counters.rs index a8b0f4a8d6d..ea1c0d2df45 100644 --- a/compiler/rustc_mir_transform/src/coverage/counters.rs +++ b/compiler/rustc_mir_transform/src/coverage/counters.rs @@ -6,6 +6,7 @@ use rustc_data_structures::graph::DirectedGraph; use rustc_index::IndexVec; use rustc_middle::bug; use rustc_middle::mir::coverage::{CounterId, CovTerm, Expression, ExpressionId, Op}; +use tracing::{debug, debug_span, instrument}; use crate::coverage::graph::{BasicCoverageBlock, CoverageGraph, TraverseCoverageGraphWithLoops}; diff --git a/compiler/rustc_mir_transform/src/coverage/graph.rs b/compiler/rustc_mir_transform/src/coverage/graph.rs index 31b20775194..0d874a6c8ba 100644 --- a/compiler/rustc_mir_transform/src/coverage/graph.rs +++ b/compiler/rustc_mir_transform/src/coverage/graph.rs @@ -10,6 +10,7 @@ use rustc_index::bit_set::BitSet; use rustc_index::IndexVec; use rustc_middle::bug; use rustc_middle::mir::{self, BasicBlock, Terminator, TerminatorKind}; +use tracing::debug; /// A coverage-specific simplification of the MIR control flow graph (CFG). The `CoverageGraph`s /// nodes are `BasicCoverageBlock`s, which encompass one or more MIR `BasicBlock`s. diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs index 96ca3b43d5c..4edba61fdec 100644 --- a/compiler/rustc_mir_transform/src/coverage/mod.rs +++ b/compiler/rustc_mir_transform/src/coverage/mod.rs @@ -13,7 +13,7 @@ use rustc_hir::intravisit::{walk_expr, Visitor}; use rustc_middle::hir::map::Map; use rustc_middle::hir::nested_filter; use rustc_middle::mir::coverage::{ - CodeRegion, CoverageKind, DecisionInfo, FunctionCoverageInfo, Mapping, MappingKind, + CoverageKind, DecisionInfo, FunctionCoverageInfo, Mapping, MappingKind, SourceRegion, }; use rustc_middle::mir::{ self, BasicBlock, BasicBlockData, SourceInfo, Statement, StatementKind, Terminator, @@ -23,18 +23,18 @@ use rustc_middle::ty::TyCtxt; use rustc_span::def_id::LocalDefId; use rustc_span::source_map::SourceMap; use rustc_span::{BytePos, Pos, RelativeBytePos, Span, Symbol}; +use tracing::{debug, debug_span, instrument, trace}; use crate::coverage::counters::{CounterIncrementSite, CoverageCounters}; use crate::coverage::graph::CoverageGraph; use crate::coverage::mappings::ExtractedMappings; -use crate::MirPass; /// Inserts `StatementKind::Coverage` statements that either instrument the binary with injected /// counters, via intrinsic `llvm.instrprof.increment`, and/or inject metadata used during codegen /// to construct the coverage map. pub struct InstrumentCoverage; -impl<'tcx> MirPass<'tcx> for InstrumentCoverage { +impl<'tcx> crate::MirPass<'tcx> for InstrumentCoverage { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { sess.instrument_coverage() } @@ -159,7 +159,7 @@ fn create_mappings<'tcx>( .expect("all BCBs with spans were given counters") .as_term() }; - let region_for_span = |span: Span| make_code_region(source_map, file_name, span, body_span); + let region_for_span = |span: Span| make_source_region(source_map, file_name, span, body_span); // Fully destructure the mappings struct to make sure we don't miss any kinds. let ExtractedMappings { @@ -175,9 +175,9 @@ fn create_mappings<'tcx>( mappings.extend(code_mappings.iter().filter_map( // Ordinary code mappings are the simplest kind. |&mappings::CodeMapping { span, bcb }| { - let code_region = region_for_span(span)?; + let source_region = region_for_span(span)?; let kind = MappingKind::Code(term_for_bcb(bcb)); - Some(Mapping { kind, code_region }) + Some(Mapping { kind, source_region }) }, )); @@ -186,29 +186,29 @@ fn create_mappings<'tcx>( let true_term = term_for_bcb(true_bcb); let false_term = term_for_bcb(false_bcb); let kind = MappingKind::Branch { true_term, false_term }; - let code_region = region_for_span(span)?; - Some(Mapping { kind, code_region }) + let source_region = region_for_span(span)?; + Some(Mapping { kind, source_region }) }, )); mappings.extend(mcdc_branches.iter().filter_map( |&mappings::MCDCBranch { span, true_bcb, false_bcb, condition_info, decision_depth: _ }| { - let code_region = region_for_span(span)?; + let source_region = region_for_span(span)?; let true_term = term_for_bcb(true_bcb); let false_term = term_for_bcb(false_bcb); let kind = match condition_info { Some(mcdc_params) => MappingKind::MCDCBranch { true_term, false_term, mcdc_params }, None => MappingKind::Branch { true_term, false_term }, }; - Some(Mapping { kind, code_region }) + Some(Mapping { kind, source_region }) }, )); mappings.extend(mcdc_decisions.iter().filter_map( |&mappings::MCDCDecision { span, bitmap_idx, num_conditions, .. }| { - let code_region = region_for_span(span)?; + let source_region = region_for_span(span)?; let kind = MappingKind::MCDCDecision(DecisionInfo { bitmap_idx, num_conditions }); - Some(Mapping { kind, code_region }) + Some(Mapping { kind, source_region }) }, )); @@ -362,19 +362,13 @@ fn inject_statement(mir_body: &mut mir::Body<'_>, counter_kind: CoverageKind, bb /// but it's hard to rule out entirely (especially in the presence of complex macros /// or other expansions), and if it does happen then skipping a span or function is /// better than an ICE or `llvm-cov` failure that the user might have no way to avoid. -fn make_code_region( +#[instrument(level = "debug", skip(source_map))] +fn make_source_region( source_map: &SourceMap, file_name: Symbol, span: Span, body_span: Span, -) -> Option<CodeRegion> { - debug!( - "Called make_code_region(file_name={}, span={}, body_span={})", - file_name, - source_map.span_to_diagnostic_string(span), - source_map.span_to_diagnostic_string(body_span) - ); - +) -> Option<SourceRegion> { let lo = span.lo(); let hi = span.hi(); @@ -424,7 +418,7 @@ fn make_code_region( start_line = source_map.doctest_offset_line(&file.name, start_line); end_line = source_map.doctest_offset_line(&file.name, end_line); - check_code_region(CodeRegion { + check_source_region(SourceRegion { file_name, start_line: start_line as u32, start_col: start_col as u32, @@ -433,12 +427,12 @@ fn make_code_region( }) } -/// If `llvm-cov` sees a code region that is improperly ordered (end < start), +/// If `llvm-cov` sees a source region that is improperly ordered (end < start), /// it will immediately exit with a fatal error. To prevent that from happening, /// discard regions that are improperly ordered, or might be interpreted in a /// way that makes them improperly ordered. -fn check_code_region(code_region: CodeRegion) -> Option<CodeRegion> { - let CodeRegion { file_name: _, start_line, start_col, end_line, end_col } = code_region; +fn check_source_region(source_region: SourceRegion) -> Option<SourceRegion> { + let SourceRegion { file_name: _, start_line, start_col, end_line, end_col } = source_region; // Line/column coordinates are supposed to be 1-based. If we ever emit // coordinates of 0, `llvm-cov` might misinterpret them. @@ -451,17 +445,17 @@ fn check_code_region(code_region: CodeRegion) -> Option<CodeRegion> { let is_ordered = (start_line, start_col) <= (end_line, end_col); if all_nonzero && end_col_has_high_bit_unset && is_ordered { - Some(code_region) + Some(source_region) } else { debug!( - ?code_region, + ?source_region, ?all_nonzero, ?end_col_has_high_bit_unset, ?is_ordered, - "Skipping code region that would be misinterpreted or rejected by LLVM" + "Skipping source region that would be misinterpreted or rejected by LLVM" ); // If this happens in a debug build, ICE to make it easier to notice. - debug_assert!(false, "Improper code region: {code_region:?}"); + debug_assert!(false, "Improper source region: {source_region:?}"); None } } diff --git a/compiler/rustc_mir_transform/src/coverage/query.rs b/compiler/rustc_mir_transform/src/coverage/query.rs index 1fce2abbbbf..e65a5fdd5e7 100644 --- a/compiler/rustc_mir_transform/src/coverage/query.rs +++ b/compiler/rustc_mir_transform/src/coverage/query.rs @@ -7,6 +7,7 @@ use rustc_middle::ty::{self, TyCtxt}; use rustc_middle::util::Providers; use rustc_span::def_id::LocalDefId; use rustc_span::sym; +use tracing::trace; /// Registers query/hook implementations related to coverage. pub(crate) fn provide(providers: &mut Providers) { diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs index 092ec1e06d2..b904b0d2599 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans.rs @@ -3,7 +3,8 @@ use std::collections::VecDeque; use rustc_data_structures::captures::Captures; use rustc_data_structures::fx::FxHashSet; use rustc_middle::mir; -use rustc_span::Span; +use rustc_span::{DesugaringKind, ExpnKind, MacroKind, Span}; +use tracing::{debug, debug_span, instrument}; use crate::coverage::graph::{BasicCoverageBlock, CoverageGraph}; use crate::coverage::spans::from_mir::{ @@ -24,7 +25,7 @@ pub(super) fn extract_refined_covspans( // First, perform the passes that need macro information. covspans.sort_by(|a, b| basic_coverage_blocks.cmp_in_dominator_order(a.bcb, b.bcb)); - remove_unwanted_macro_spans(&mut covspans); + remove_unwanted_expansion_spans(&mut covspans); split_visible_macro_spans(&mut covspans); // We no longer need the extra information in `SpanFromMir`, so convert to `Covspan`. @@ -75,18 +76,24 @@ pub(super) fn extract_refined_covspans( /// invocation, which is unhelpful. Keeping only the first such span seems to /// give better mappings, so remove the others. /// +/// Similarly, `await` expands to a branch on the discriminant of `Poll`, which +/// leads to incorrect coverage if the `Future` is immediately ready (#98712). +/// /// (The input spans should be sorted in BCB dominator order, so that the /// retained "first" span is likely to dominate the others.) -fn remove_unwanted_macro_spans(covspans: &mut Vec<SpanFromMir>) { - let mut seen_macro_spans = FxHashSet::default(); +fn remove_unwanted_expansion_spans(covspans: &mut Vec<SpanFromMir>) { + let mut deduplicated_spans = FxHashSet::default(); + covspans.retain(|covspan| { - // Ignore (retain) non-macro-expansion spans. - if covspan.visible_macro.is_none() { - return true; + match covspan.expn_kind { + // Retain only the first await-related or macro-expanded covspan with this span. + Some(ExpnKind::Desugaring(kind)) if kind == DesugaringKind::Await => { + deduplicated_spans.insert(covspan.span) + } + Some(ExpnKind::Macro(MacroKind::Bang, _)) => deduplicated_spans.insert(covspan.span), + // Ignore (retain) other spans. + _ => true, } - - // Retain only the first macro-expanded covspan with this span. - seen_macro_spans.insert(covspan.span) }); } @@ -98,7 +105,9 @@ fn split_visible_macro_spans(covspans: &mut Vec<SpanFromMir>) { let mut extra_spans = vec![]; covspans.retain(|covspan| { - let Some(visible_macro) = covspan.visible_macro else { return true }; + let Some(ExpnKind::Macro(MacroKind::Bang, visible_macro)) = covspan.expn_kind else { + return true; + }; let split_len = visible_macro.as_str().len() as u32 + 1; let (before, after) = covspan.span.split_at(split_len); @@ -110,8 +119,8 @@ fn split_visible_macro_spans(covspans: &mut Vec<SpanFromMir>) { return true; } - extra_spans.push(SpanFromMir::new(before, covspan.visible_macro, covspan.bcb)); - extra_spans.push(SpanFromMir::new(after, covspan.visible_macro, covspan.bcb)); + extra_spans.push(SpanFromMir::new(before, covspan.expn_kind.clone(), covspan.bcb)); + extra_spans.push(SpanFromMir::new(after, covspan.expn_kind.clone(), covspan.bcb)); false // Discard the original covspan that we just split. }); diff --git a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs index 32bd25bf4b9..7f5765c9462 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs @@ -3,13 +3,13 @@ use rustc_middle::mir::coverage::CoverageKind; use rustc_middle::mir::{ self, FakeReadCause, Statement, StatementKind, Terminator, TerminatorKind, }; -use rustc_span::{Span, Symbol}; +use rustc_span::{ExpnKind, Span}; use crate::coverage::graph::{ BasicCoverageBlock, BasicCoverageBlockData, CoverageGraph, START_BCB, }; use crate::coverage::spans::Covspan; -use crate::coverage::unexpand::unexpand_into_body_span_with_visible_macro; +use crate::coverage::unexpand::unexpand_into_body_span_with_expn_kind; use crate::coverage::ExtractedHirInfo; pub(crate) struct ExtractedCovspans { @@ -60,7 +60,7 @@ fn bcb_to_initial_coverage_spans<'a, 'tcx>( let data = &mir_body[bb]; let unexpand = move |expn_span| { - unexpand_into_body_span_with_visible_macro(expn_span, body_span) + unexpand_into_body_span_with_expn_kind(expn_span, body_span) // Discard any spans that fill the entire body, because they tend // to represent compiler-inserted code, e.g. implicitly returning `()`. .filter(|(span, _)| !span.source_equal(body_span)) @@ -68,9 +68,9 @@ fn bcb_to_initial_coverage_spans<'a, 'tcx>( let mut extract_statement_span = |statement| { let expn_span = filtered_statement_span(statement)?; - let (span, visible_macro) = unexpand(expn_span)?; + let (span, expn_kind) = unexpand(expn_span)?; - initial_covspans.push(SpanFromMir::new(span, visible_macro, bcb)); + initial_covspans.push(SpanFromMir::new(span, expn_kind, bcb)); Some(()) }; for statement in data.statements.iter() { @@ -79,9 +79,9 @@ fn bcb_to_initial_coverage_spans<'a, 'tcx>( let mut extract_terminator_span = |terminator| { let expn_span = filtered_terminator_span(terminator)?; - let (span, visible_macro) = unexpand(expn_span)?; + let (span, expn_kind) = unexpand(expn_span)?; - initial_covspans.push(SpanFromMir::new(span, visible_macro, bcb)); + initial_covspans.push(SpanFromMir::new(span, expn_kind, bcb)); Some(()) }; extract_terminator_span(data.terminator()); @@ -214,7 +214,7 @@ pub(crate) struct SpanFromMir { /// With the exception of `fn_sig_span`, this should always be contained /// within `body_span`. pub(crate) span: Span, - pub(crate) visible_macro: Option<Symbol>, + pub(crate) expn_kind: Option<ExpnKind>, pub(crate) bcb: BasicCoverageBlock, } @@ -223,12 +223,12 @@ impl SpanFromMir { Self::new(fn_sig_span, None, START_BCB) } - pub(crate) fn new(span: Span, visible_macro: Option<Symbol>, bcb: BasicCoverageBlock) -> Self { - Self { span, visible_macro, bcb } + pub(crate) fn new(span: Span, expn_kind: Option<ExpnKind>, bcb: BasicCoverageBlock) -> Self { + Self { span, expn_kind, bcb } } pub(crate) fn into_covspan(self) -> Covspan { - let Self { span, visible_macro: _, bcb } = self; + let Self { span, expn_kind: _, bcb } = self; Covspan { span, bcb } } } diff --git a/compiler/rustc_mir_transform/src/coverage/unexpand.rs b/compiler/rustc_mir_transform/src/coverage/unexpand.rs index 8cde291b907..cb861544736 100644 --- a/compiler/rustc_mir_transform/src/coverage/unexpand.rs +++ b/compiler/rustc_mir_transform/src/coverage/unexpand.rs @@ -1,4 +1,4 @@ -use rustc_span::{ExpnKind, MacroKind, Span, Symbol}; +use rustc_span::{ExpnKind, Span}; /// Walks through the expansion ancestors of `original_span` to find a span that /// is contained in `body_span` and has the same [syntax context] as `body_span`. @@ -13,20 +13,15 @@ pub(crate) fn unexpand_into_body_span(original_span: Span, body_span: Span) -> O /// /// If the returned span represents a bang-macro invocation (e.g. `foo!(..)`), /// the returned symbol will be the name of that macro (e.g. `foo`). -pub(crate) fn unexpand_into_body_span_with_visible_macro( +pub(crate) fn unexpand_into_body_span_with_expn_kind( original_span: Span, body_span: Span, -) -> Option<(Span, Option<Symbol>)> { +) -> Option<(Span, Option<ExpnKind>)> { let (span, prev) = unexpand_into_body_span_with_prev(original_span, body_span)?; - let visible_macro = prev - .map(|prev| match prev.ctxt().outer_expn_data().kind { - ExpnKind::Macro(MacroKind::Bang, name) => Some(name), - _ => None, - }) - .flatten(); + let expn_kind = prev.map(|prev| prev.ctxt().outer_expn_data().kind); - Some((span, visible_macro)) + Some((span, expn_kind)) } /// Walks through the expansion ancestors of `original_span` to find a span that diff --git a/compiler/rustc_mir_transform/src/ctfe_limit.rs b/compiler/rustc_mir_transform/src/ctfe_limit.rs index ff9fc776e54..ea473b64ce5 100644 --- a/compiler/rustc_mir_transform/src/ctfe_limit.rs +++ b/compiler/rustc_mir_transform/src/ctfe_limit.rs @@ -6,12 +6,11 @@ use rustc_middle::mir::{ BasicBlock, BasicBlockData, Body, Statement, StatementKind, TerminatorKind, }; use rustc_middle::ty::TyCtxt; - -use crate::MirPass; +use tracing::instrument; pub struct CtfeLimit; -impl<'tcx> MirPass<'tcx> for CtfeLimit { +impl<'tcx> crate::MirPass<'tcx> for CtfeLimit { #[instrument(skip(self, _tcx, body))] fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let doms = body.basic_blocks.dominators(); diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs index 0fc4d6b9f4e..46f7408ef80 100644 --- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs +++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs @@ -19,6 +19,7 @@ use rustc_mir_dataflow::value_analysis::{ use rustc_mir_dataflow::{Analysis, Results, ResultsVisitor}; use rustc_span::DUMMY_SP; use rustc_target::abi::{Abi, FieldIdx, Size, VariantIdx, FIRST_VARIANT}; +use tracing::{debug, debug_span, instrument}; // These constants are somewhat random guesses and have not been optimized. // If `tcx.sess.mir_opt_level() >= 4`, we ignore the limits (this can become very expensive). @@ -27,7 +28,7 @@ const PLACE_LIMIT: usize = 100; pub struct DataflowConstProp; -impl<'tcx> MirPass<'tcx> for DataflowConstProp { +impl<'tcx> crate::MirPass<'tcx> for DataflowConstProp { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { sess.mir_opt_level() >= 3 } @@ -338,7 +339,7 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { tcx, local_decls: &body.local_decls, ecx: InterpCx::new(tcx, DUMMY_SP, param_env, DummyMachine), - param_env: param_env, + param_env, } } @@ -382,7 +383,7 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { place: PlaceIndex, mut operand: OpTy<'tcx>, projection: &[PlaceElem<'tcx>], - ) -> Option<!> { + ) { for &(mut proj_elem) in projection { if let PlaceElem::Index(index) = proj_elem { if let FlatSet::Elem(index) = state.get(index.into(), &self.map) @@ -391,10 +392,14 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { { proj_elem = PlaceElem::ConstantIndex { offset, min_length, from_end: false }; } else { - return None; + return; } } - operand = self.ecx.project(&operand, proj_elem).ok()?; + operand = if let Ok(operand) = self.ecx.project(&operand, proj_elem) { + operand + } else { + return; + } } self.map.for_each_projection_value( @@ -426,8 +431,6 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { } }, ); - - None } fn binary_op( diff --git a/compiler/rustc_mir_transform/src/dead_store_elimination.rs b/compiler/rustc_mir_transform/src/dead_store_elimination.rs index f473073083a..9081a2e2e30 100644 --- a/compiler/rustc_mir_transform/src/dead_store_elimination.rs +++ b/compiler/rustc_mir_transform/src/dead_store_elimination.rs @@ -32,7 +32,7 @@ pub fn eliminate<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let borrowed_locals = borrowed_locals(body); // If the user requests complete debuginfo, mark the locals that appear in it as live, so - // we don't remove assignements to them. + // we don't remove assignments to them. let mut always_live = debuginfo_locals(body); always_live.union(&borrowed_locals); @@ -132,7 +132,7 @@ pub enum DeadStoreElimination { Final, } -impl<'tcx> MirPass<'tcx> for DeadStoreElimination { +impl<'tcx> crate::MirPass<'tcx> for DeadStoreElimination { fn name(&self) -> &'static str { match self { DeadStoreElimination::Initial => "DeadStoreElimination-initial", diff --git a/compiler/rustc_mir_transform/src/deduplicate_blocks.rs b/compiler/rustc_mir_transform/src/deduplicate_blocks.rs index 4a94c3eca86..be50c1da8a4 100644 --- a/compiler/rustc_mir_transform/src/deduplicate_blocks.rs +++ b/compiler/rustc_mir_transform/src/deduplicate_blocks.rs @@ -9,12 +9,13 @@ use rustc_data_structures::fx::FxHashMap; use rustc_middle::mir::visit::MutVisitor; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; +use tracing::debug; use super::simplify::simplify_cfg; pub struct DeduplicateBlocks; -impl<'tcx> MirPass<'tcx> for DeduplicateBlocks { +impl<'tcx> crate::MirPass<'tcx> for DeduplicateBlocks { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { sess.mir_opt_level() >= 4 } diff --git a/compiler/rustc_mir_transform/src/deref_separator.rs b/compiler/rustc_mir_transform/src/deref_separator.rs index 0e2fccc85da..a878f777448 100644 --- a/compiler/rustc_mir_transform/src/deref_separator.rs +++ b/compiler/rustc_mir_transform/src/deref_separator.rs @@ -78,7 +78,7 @@ pub fn deref_finder<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { checker.patcher.apply(body); } -impl<'tcx> MirPass<'tcx> for Derefer { +impl<'tcx> crate::MirPass<'tcx> for Derefer { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { deref_finder(tcx, body); } diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index ed924761892..67bee36b8a5 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -38,7 +38,7 @@ //! not contain any indirection through a pointer or any indexing projections. //! //! * `p` and `q` must have the **same type**. If we replace a local with a subtype or supertype, -//! we may end up with a differnet vtable for that local. See the `subtyping-impacts-selection` +//! we may end up with a different vtable for that local. See the `subtyping-impacts-selection` //! tests for an example where that causes issues. //! //! * We need to make sure that the goal of "merging the memory" is actually structurally possible @@ -144,12 +144,11 @@ use rustc_middle::ty::TyCtxt; use rustc_mir_dataflow::impls::MaybeLiveLocals; use rustc_mir_dataflow::points::{save_as_intervals, DenseLocationMap, PointIndex}; use rustc_mir_dataflow::Analysis; - -use crate::MirPass; +use tracing::{debug, trace}; pub struct DestinationPropagation; -impl<'tcx> MirPass<'tcx> for DestinationPropagation { +impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { // For now, only run at MIR opt level 3. Two things need to be changed before this can be // turned on by default: @@ -164,7 +163,8 @@ impl<'tcx> MirPass<'tcx> for DestinationPropagation { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let def_id = body.source.def_id(); - let mut allocations = Allocations::default(); + let mut candidates = Candidates::default(); + let mut write_info = WriteInfo::default(); trace!(func = ?tcx.def_path_str(def_id)); let borrowed = rustc_mir_dataflow::impls::borrowed_locals(body); @@ -192,12 +192,7 @@ impl<'tcx> MirPass<'tcx> for DestinationPropagation { loop { // PERF: Can we do something smarter than recalculating the candidates and liveness // results? - let mut candidates = find_candidates( - body, - &borrowed, - &mut allocations.candidates, - &mut allocations.candidates_reverse, - ); + candidates.reset_and_find(body, &borrowed); trace!(?candidates); dest_prop_mir_dump(tcx, body, &points, &live, round_count); @@ -205,7 +200,7 @@ impl<'tcx> MirPass<'tcx> for DestinationPropagation { &mut candidates, &points, &live, - &mut allocations.write_info, + &mut write_info, body, ); @@ -254,20 +249,8 @@ impl<'tcx> MirPass<'tcx> for DestinationPropagation { } } -/// Container for the various allocations that we need. -/// -/// We store these here and hand out `&mut` access to them, instead of dropping and recreating them -/// frequently. Everything with a `&'alloc` lifetime points into here. -#[derive(Default)] -struct Allocations { - candidates: FxIndexMap<Local, Vec<Local>>, - candidates_reverse: FxIndexMap<Local, Vec<Local>>, - write_info: WriteInfo, - // PERF: Do this for `MaybeLiveLocals` allocations too. -} - -#[derive(Debug)] -struct Candidates<'alloc> { +#[derive(Debug, Default)] +struct Candidates { /// The set of candidates we are considering in this optimization. /// /// We will always merge the key into at most one of its values. @@ -282,11 +265,12 @@ struct Candidates<'alloc> { /// /// We will still report that we would like to merge `_1` and `_2` in an attempt to allow us to /// remove that assignment. - c: &'alloc mut FxIndexMap<Local, Vec<Local>>, + c: FxIndexMap<Local, Vec<Local>>, + /// A reverse index of the `c` set; if the `c` set contains `a => Place { local: b, proj }`, /// then this contains `b => a`. // PERF: Possibly these should be `SmallVec`s? - reverse: &'alloc mut FxIndexMap<Local, Vec<Local>>, + reverse: FxIndexMap<Local, Vec<Local>>, } ////////////////////////////////////////////////////////// @@ -359,19 +343,40 @@ impl<'a, 'tcx> MutVisitor<'tcx> for Merger<'a, 'tcx> { // // This section enforces bullet point 2 -struct FilterInformation<'a, 'body, 'alloc, 'tcx> { - body: &'body Body<'tcx>, +struct FilterInformation<'a, 'tcx> { + body: &'a Body<'tcx>, points: &'a DenseLocationMap, live: &'a SparseIntervalMatrix<Local, PointIndex>, - candidates: &'a mut Candidates<'alloc>, - write_info: &'alloc mut WriteInfo, + candidates: &'a mut Candidates, + write_info: &'a mut WriteInfo, at: Location, } // We first implement some utility functions which we will expose removing candidates according to // different needs. Throughout the liveness filtering, the `candidates` are only ever accessed // through these methods, and not directly. -impl<'alloc> Candidates<'alloc> { +impl Candidates { + /// Collects the candidates for merging. + /// + /// This is responsible for enforcing the first and third bullet point. + fn reset_and_find<'tcx>(&mut self, body: &Body<'tcx>, borrowed: &BitSet<Local>) { + self.c.clear(); + self.reverse.clear(); + let mut visitor = FindAssignments { body, candidates: &mut self.c, borrowed }; + visitor.visit_body(body); + // Deduplicate candidates. + for (_, cands) in self.c.iter_mut() { + cands.sort(); + cands.dedup(); + } + // Generate the reverse map. + for (src, cands) in self.c.iter() { + for dest in cands.iter().copied() { + self.reverse.entry(dest).or_default().push(*src); + } + } + } + /// Just `Vec::retain`, but the condition is inverted and we add debugging output fn vec_filter_candidates( src: Local, @@ -446,7 +451,7 @@ enum CandidateFilter { Remove, } -impl<'a, 'body, 'alloc, 'tcx> FilterInformation<'a, 'body, 'alloc, 'tcx> { +impl<'a, 'tcx> FilterInformation<'a, 'tcx> { /// Filters the set of candidates to remove those that conflict. /// /// The steps we take are exactly those that are outlined at the top of the file. For each @@ -464,12 +469,12 @@ impl<'a, 'body, 'alloc, 'tcx> FilterInformation<'a, 'body, 'alloc, 'tcx> { /// before the statement/terminator will correctly report locals that are read in the /// statement/terminator to be live. We are additionally conservative by treating all written to /// locals as also being read from. - fn filter_liveness<'b>( - candidates: &mut Candidates<'alloc>, + fn filter_liveness( + candidates: &mut Candidates, points: &DenseLocationMap, live: &SparseIntervalMatrix<Local, PointIndex>, - write_info_alloc: &'alloc mut WriteInfo, - body: &'b Body<'tcx>, + write_info: &mut WriteInfo, + body: &Body<'tcx>, ) { let mut this = FilterInformation { body, @@ -478,7 +483,7 @@ impl<'a, 'body, 'alloc, 'tcx> FilterInformation<'a, 'body, 'alloc, 'tcx> { candidates, // We don't actually store anything at this scope, we just keep things here to be able // to reuse the allocation. - write_info: write_info_alloc, + write_info, // Doesn't matter what we put here, will be overwritten before being used at: Location::START, }; @@ -735,40 +740,13 @@ fn places_to_candidate_pair<'tcx>( Some((a, b)) } -/// Collects the candidates for merging -/// -/// This is responsible for enforcing the first and third bullet point. -fn find_candidates<'alloc, 'tcx>( - body: &Body<'tcx>, - borrowed: &BitSet<Local>, - candidates: &'alloc mut FxIndexMap<Local, Vec<Local>>, - candidates_reverse: &'alloc mut FxIndexMap<Local, Vec<Local>>, -) -> Candidates<'alloc> { - candidates.clear(); - candidates_reverse.clear(); - let mut visitor = FindAssignments { body, candidates, borrowed }; - visitor.visit_body(body); - // Deduplicate candidates - for (_, cands) in candidates.iter_mut() { - cands.sort(); - cands.dedup(); - } - // Generate the reverse map - for (src, cands) in candidates.iter() { - for dest in cands.iter().copied() { - candidates_reverse.entry(dest).or_default().push(*src); - } - } - Candidates { c: candidates, reverse: candidates_reverse } -} - -struct FindAssignments<'a, 'alloc, 'tcx> { +struct FindAssignments<'a, 'tcx> { body: &'a Body<'tcx>, - candidates: &'alloc mut FxIndexMap<Local, Vec<Local>>, + candidates: &'a mut FxIndexMap<Local, Vec<Local>>, borrowed: &'a BitSet<Local>, } -impl<'tcx> Visitor<'tcx> for FindAssignments<'_, '_, 'tcx> { +impl<'tcx> Visitor<'tcx> for FindAssignments<'_, 'tcx> { fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) { if let StatementKind::Assign(box ( lhs, @@ -820,9 +798,9 @@ fn is_local_required(local: Local, body: &Body<'_>) -> bool { ///////////////////////////////////////////////////////// // MIR Dump -fn dest_prop_mir_dump<'body, 'tcx>( +fn dest_prop_mir_dump<'tcx>( tcx: TyCtxt<'tcx>, - body: &'body Body<'tcx>, + body: &Body<'tcx>, points: &DenseLocationMap, live: &SparseIntervalMatrix<Local, PointIndex>, round: usize, diff --git a/compiler/rustc_mir_transform/src/dump_mir.rs b/compiler/rustc_mir_transform/src/dump_mir.rs index 29db45f9450..06ae1b490d7 100644 --- a/compiler/rustc_mir_transform/src/dump_mir.rs +++ b/compiler/rustc_mir_transform/src/dump_mir.rs @@ -7,11 +7,9 @@ use rustc_middle::mir::{write_mir_pretty, Body}; use rustc_middle::ty::TyCtxt; use rustc_session::config::{OutFileName, OutputType}; -use crate::MirPass; - pub struct Marker(pub &'static str); -impl<'tcx> MirPass<'tcx> for Marker { +impl<'tcx> crate::MirPass<'tcx> for Marker { fn name(&self) -> &'static str { self.0 } diff --git a/compiler/rustc_mir_transform/src/early_otherwise_branch.rs b/compiler/rustc_mir_transform/src/early_otherwise_branch.rs index 49e41c35f1f..1c54cd70023 100644 --- a/compiler/rustc_mir_transform/src/early_otherwise_branch.rs +++ b/compiler/rustc_mir_transform/src/early_otherwise_branch.rs @@ -3,6 +3,7 @@ use std::fmt::Debug; use rustc_middle::mir::patch::MirPatch; use rustc_middle::mir::*; use rustc_middle::ty::{Ty, TyCtxt}; +use tracing::trace; use super::simplify::simplify_cfg; @@ -91,7 +92,7 @@ use super::simplify::simplify_cfg; /// ``` pub struct EarlyOtherwiseBranch; -impl<'tcx> MirPass<'tcx> for EarlyOtherwiseBranch { +impl<'tcx> crate::MirPass<'tcx> for EarlyOtherwiseBranch { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { sess.mir_opt_level() >= 2 } diff --git a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs index e5778f8a05d..66e49d556e2 100644 --- a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs +++ b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs @@ -62,11 +62,13 @@ impl<'tcx, 'a> MutVisitor<'tcx> for ElaborateBoxDerefVisitor<'tcx, 'a> { let base_ty = self.local_decls[place.local].ty; // Derefer ensures that derefs are always the first projection - if place.projection.first() == Some(&PlaceElem::Deref) && base_ty.is_box() { + if let Some(PlaceElem::Deref) = place.projection.first() + && let Some(boxed_ty) = base_ty.boxed_ty() + { let source_info = self.local_decls[place.local].source_info; let (unique_ty, nonnull_ty, ptr_ty) = - build_ptr_tys(tcx, base_ty.boxed_ty(), self.unique_did, self.nonnull_did); + build_ptr_tys(tcx, boxed_ty, self.unique_did, self.nonnull_did); let ptr_local = self.patch.new_temp(ptr_ty, source_info.span); @@ -88,7 +90,7 @@ impl<'tcx, 'a> MutVisitor<'tcx> for ElaborateBoxDerefVisitor<'tcx, 'a> { pub struct ElaborateBoxDerefs; -impl<'tcx> MirPass<'tcx> for ElaborateBoxDerefs { +impl<'tcx> crate::MirPass<'tcx> for ElaborateBoxDerefs { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { if let Some(def_id) = tcx.lang_items().owned_box() { let unique_did = tcx.adt_def(def_id).non_enum_variant().fields[FieldIdx::ZERO].did; @@ -120,13 +122,15 @@ impl<'tcx> MirPass<'tcx> for ElaborateBoxDerefs { for (base, elem) in place.iter_projections() { let base_ty = base.ty(&body.local_decls, tcx).ty; - if elem == PlaceElem::Deref && base_ty.is_box() { + if let PlaceElem::Deref = elem + && let Some(boxed_ty) = base_ty.boxed_ty() + { // Clone the projections before us, since now we need to mutate them. let new_projections = new_projections.get_or_insert_with(|| base.projection.to_vec()); let (unique_ty, nonnull_ty, ptr_ty) = - build_ptr_tys(tcx, base_ty.boxed_ty(), unique_did, nonnull_did); + build_ptr_tys(tcx, boxed_ty, unique_did, nonnull_did); new_projections.extend_from_slice(&build_projection( unique_ty, nonnull_ty, ptr_ty, diff --git a/compiler/rustc_mir_transform/src/elaborate_drops.rs b/compiler/rustc_mir_transform/src/elaborate_drops.rs index 5a22ef77903..f4a951ebde6 100644 --- a/compiler/rustc_mir_transform/src/elaborate_drops.rs +++ b/compiler/rustc_mir_transform/src/elaborate_drops.rs @@ -15,6 +15,7 @@ use rustc_mir_dataflow::{ }; use rustc_span::Span; use rustc_target::abi::{FieldIdx, VariantIdx}; +use tracing::{debug, instrument}; use crate::deref_separator::deref_finder; @@ -48,7 +49,7 @@ use crate::deref_separator::deref_finder; /// ``` pub struct ElaborateDrops; -impl<'tcx> MirPass<'tcx> for ElaborateDrops { +impl<'tcx> crate::MirPass<'tcx> for ElaborateDrops { #[instrument(level = "trace", skip(self, tcx, body))] fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { debug!("elaborate_drops({:?} @ {:?})", body.source, body.span); diff --git a/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs b/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs index 9a2cc057232..81875e3d012 100644 --- a/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs +++ b/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs @@ -6,6 +6,7 @@ use rustc_middle::{bug, span_bug}; use rustc_session::lint::builtin::FFI_UNWIND_CALLS; use rustc_target::spec::abi::Abi; use rustc_target::spec::PanicStrategy; +use tracing::debug; use crate::errors; diff --git a/compiler/rustc_mir_transform/src/function_item_references.rs b/compiler/rustc_mir_transform/src/function_item_references.rs index b7873e73c18..199fd0f10ee 100644 --- a/compiler/rustc_mir_transform/src/function_item_references.rs +++ b/compiler/rustc_mir_transform/src/function_item_references.rs @@ -9,11 +9,11 @@ use rustc_span::symbol::sym; use rustc_span::Span; use rustc_target::spec::abi::Abi; -use crate::{errors, MirLint}; +use crate::errors; pub struct FunctionItemReferences; -impl<'tcx> MirLint<'tcx> for FunctionItemReferences { +impl<'tcx> crate::MirLint<'tcx> for FunctionItemReferences { fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) { let mut checker = FunctionItemRefChecker { tcx, body }; checker.visit_body(body); diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs index 90e3ba26a43..df0fcc42e59 100644 --- a/compiler/rustc_mir_transform/src/gvn.rs +++ b/compiler/rustc_mir_transform/src/gvn.rs @@ -105,12 +105,13 @@ use rustc_span::def_id::DefId; use rustc_span::DUMMY_SP; use rustc_target::abi::{self, Abi, FieldIdx, Size, VariantIdx, FIRST_VARIANT}; use smallvec::SmallVec; +use tracing::{debug, instrument, trace}; use crate::ssa::{AssignedValue, SsaLocals}; pub struct GVN; -impl<'tcx> MirPass<'tcx> for GVN { +impl<'tcx> crate::MirPass<'tcx> for GVN { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { sess.mir_opt_level() >= 2 } diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 61fc5fc8816..6cc7e0ee1e4 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -20,6 +20,7 @@ use rustc_span::source_map::Spanned; use rustc_span::sym; use rustc_target::abi::FieldIdx; use rustc_target::spec::abi::Abi; +use tracing::{debug, instrument, trace, trace_span}; use crate::cost_checker::CostChecker; use crate::deref_separator::deref_finder; @@ -41,7 +42,7 @@ struct CallSite<'tcx> { source_info: SourceInfo, } -impl<'tcx> MirPass<'tcx> for Inline { +impl<'tcx> crate::MirPass<'tcx> for Inline { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { // FIXME(#127234): Coverage instrumentation currently doesn't handle inlined // MIR correctly when Modified Condition/Decision Coverage is enabled. @@ -341,7 +342,6 @@ impl<'tcx> Inliner<'tcx> { | InstanceKind::FnPtrShim(..) | InstanceKind::ClosureOnceShim { .. } | InstanceKind::ConstructCoroutineInClosureShim { .. } - | InstanceKind::CoroutineKindShim { .. } | InstanceKind::DropGlue(..) | InstanceKind::CloneShim(..) | InstanceKind::ThreadLocalShim(..) diff --git a/compiler/rustc_mir_transform/src/inline/cycle.rs b/compiler/rustc_mir_transform/src/inline/cycle.rs index f5274c664cf..9828e90de88 100644 --- a/compiler/rustc_mir_transform/src/inline/cycle.rs +++ b/compiler/rustc_mir_transform/src/inline/cycle.rs @@ -5,6 +5,7 @@ use rustc_middle::mir::TerminatorKind; use rustc_middle::ty::{self, GenericArgsRef, InstanceKind, TyCtxt, TypeVisitableExt}; use rustc_session::Limit; use rustc_span::sym; +use tracing::{instrument, trace}; // FIXME: check whether it is cheaper to precompute the entire call graph instead of invoking // this query ridiculously often. @@ -88,7 +89,6 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( | InstanceKind::FnPtrShim(..) | InstanceKind::ClosureOnceShim { .. } | InstanceKind::ConstructCoroutineInClosureShim { .. } - | InstanceKind::CoroutineKindShim { .. } | InstanceKind::ThreadLocalShim { .. } | InstanceKind::CloneShim(..) => {} @@ -136,6 +136,14 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( } false } + // FIXME(-Znext-solver): Remove this hack when trait solver overflow can return an error. + // In code like that pointed out in #128887, the type complexity we ask the solver to deal with + // grows as we recurse into the call graph. If we use the same recursion limit here and in the + // solver, the solver hits the limit first and emits a fatal error. But if we use a reduced + // limit, we will hit the limit first and give up on looking for inlining. And in any case, + // the default recursion limits are quite generous for us. If we need to recurse 64 times + // into the call graph, we're probably not going to find any useful MIR inlining. + let recursion_limit = tcx.recursion_limit() / 2; process( tcx, param_env, @@ -144,7 +152,7 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( &mut Vec::new(), &mut FxHashSet::default(), &mut FxHashMap::default(), - tcx.recursion_limit(), + recursion_limit, ) } diff --git a/compiler/rustc_mir_transform/src/instsimplify.rs b/compiler/rustc_mir_transform/src/instsimplify.rs index 3ec553d0ba0..4fbfa744e67 100644 --- a/compiler/rustc_mir_transform/src/instsimplify.rs +++ b/compiler/rustc_mir_transform/src/instsimplify.rs @@ -27,7 +27,7 @@ impl InstSimplify { } } -impl<'tcx> MirPass<'tcx> for InstSimplify { +impl<'tcx> crate::MirPass<'tcx> for InstSimplify { fn name(&self) -> &'static str { self.name() } diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index 96c52845a4a..02dd56e1b4f 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -51,6 +51,7 @@ use rustc_mir_dataflow::lattice::HasBottom; use rustc_mir_dataflow::value_analysis::{Map, PlaceIndex, State, TrackElem}; use rustc_span::DUMMY_SP; use rustc_target::abi::{TagEncoding, Variants}; +use tracing::{debug, instrument, trace}; use crate::cost_checker::CostChecker; @@ -60,7 +61,7 @@ const MAX_BACKTRACK: usize = 5; const MAX_COST: usize = 100; const MAX_PLACES: usize = 100; -impl<'tcx> MirPass<'tcx> for JumpThreading { +impl<'tcx> crate::MirPass<'tcx> for JumpThreading { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { sess.mir_opt_level() >= 2 } @@ -191,26 +192,26 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { /// Recursion entry point to find threading opportunities. #[instrument(level = "trace", skip(self))] - fn start_from_switch(&mut self, bb: BasicBlock) -> Option<!> { + fn start_from_switch(&mut self, bb: BasicBlock) { let bbdata = &self.body[bb]; if bbdata.is_cleanup || self.loop_headers.contains(bb) { - return None; + return; } - let (discr, targets) = bbdata.terminator().kind.as_switch()?; - let discr = discr.place()?; + let Some((discr, targets)) = bbdata.terminator().kind.as_switch() else { return }; + let Some(discr) = discr.place() else { return }; debug!(?discr, ?bb); let discr_ty = discr.ty(self.body, self.tcx).ty; - let discr_layout = self.ecx.layout_of(discr_ty).ok()?; + let Ok(discr_layout) = self.ecx.layout_of(discr_ty) else { return }; - let discr = self.map.find(discr.as_ref())?; + let Some(discr) = self.map.find(discr.as_ref()) else { return }; debug!(?discr); let cost = CostChecker::new(self.tcx, self.param_env, None, self.body); let mut state = State::new_reachable(); let conds = if let Some((value, then, else_)) = targets.as_static_if() { - let value = ScalarInt::try_from_uint(value, discr_layout.size)?; + let Some(value) = ScalarInt::try_from_uint(value, discr_layout.size) else { return }; self.arena.alloc_from_iter([ Condition { value, polarity: Polarity::Eq, target: then }, Condition { value, polarity: Polarity::Ne, target: else_ }, @@ -225,7 +226,6 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { state.insert_value_idx(discr, conds, self.map); self.find_opportunity(bb, state, cost, 0); - None } /// Recursively walk statements backwards from this bb's terminator to find threading @@ -364,18 +364,17 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { lhs: PlaceIndex, rhs: ImmTy<'tcx>, state: &mut State<ConditionSet<'a>>, - ) -> Option<!> { + ) { let register_opportunity = |c: Condition| { debug!(?bb, ?c.target, "register"); self.opportunities.push(ThreadingOpportunity { chain: vec![bb], target: c.target }) }; - let conditions = state.try_get_idx(lhs, self.map)?; - if let Immediate::Scalar(Scalar::Int(int)) = *rhs { + if let Some(conditions) = state.try_get_idx(lhs, self.map) + && let Immediate::Scalar(Scalar::Int(int)) = *rhs + { conditions.iter_matches(int).for_each(register_opportunity); } - - None } /// If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`. @@ -428,22 +427,23 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { lhs: PlaceIndex, rhs: &Operand<'tcx>, state: &mut State<ConditionSet<'a>>, - ) -> Option<!> { + ) { match rhs { // If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`. Operand::Constant(constant) => { - let constant = - self.ecx.eval_mir_constant(&constant.const_, constant.span, None).ok()?; + let Ok(constant) = + self.ecx.eval_mir_constant(&constant.const_, constant.span, None) + else { + return; + }; self.process_constant(bb, lhs, constant, state); } // Transfer the conditions on the copied rhs. Operand::Move(rhs) | Operand::Copy(rhs) => { - let rhs = self.map.find(rhs.as_ref())?; + let Some(rhs) = self.map.find(rhs.as_ref()) else { return }; state.insert_place_idx(rhs, lhs, self.map); } } - - None } #[instrument(level = "trace", skip(self))] @@ -453,16 +453,14 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { lhs_place: &Place<'tcx>, rhs: &Rvalue<'tcx>, state: &mut State<ConditionSet<'a>>, - ) -> Option<!> { - let lhs = self.map.find(lhs_place.as_ref())?; + ) { + let Some(lhs) = self.map.find(lhs_place.as_ref()) else { return }; match rhs { - Rvalue::Use(operand) => self.process_operand(bb, lhs, operand, state)?, + Rvalue::Use(operand) => self.process_operand(bb, lhs, operand, state), // Transfer the conditions on the copy rhs. - Rvalue::CopyForDeref(rhs) => { - self.process_operand(bb, lhs, &Operand::Copy(*rhs), state)? - } + Rvalue::CopyForDeref(rhs) => self.process_operand(bb, lhs, &Operand::Copy(*rhs), state), Rvalue::Discriminant(rhs) => { - let rhs = self.map.find_discr(rhs.as_ref())?; + let Some(rhs) = self.map.find_discr(rhs.as_ref()) else { return }; state.insert_place_idx(rhs, lhs, self.map); } // If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`. @@ -470,7 +468,7 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { let agg_ty = lhs_place.ty(self.body, self.tcx).ty; let lhs = match kind { // Do not support unions. - AggregateKind::Adt(.., Some(_)) => return None, + AggregateKind::Adt(.., Some(_)) => return, AggregateKind::Adt(_, variant_index, ..) if agg_ty.is_enum() => { if let Some(discr_target) = self.map.apply(lhs, TrackElem::Discriminant) && let Ok(discr_value) = @@ -478,7 +476,11 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { { self.process_immediate(bb, discr_target, discr_value, state); } - self.map.apply(lhs, TrackElem::Variant(*variant_index))? + if let Some(idx) = self.map.apply(lhs, TrackElem::Variant(*variant_index)) { + idx + } else { + return; + } } _ => lhs, }; @@ -490,8 +492,8 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { } // Transfer the conditions on the copy rhs, after inversing polarity. Rvalue::UnaryOp(UnOp::Not, Operand::Move(place) | Operand::Copy(place)) => { - let conditions = state.try_get_idx(lhs, self.map)?; - let place = self.map.find(place.as_ref())?; + let Some(conditions) = state.try_get_idx(lhs, self.map) else { return }; + let Some(place) = self.map.find(place.as_ref()) else { return }; let conds = conditions.map(self.arena, Condition::inv); state.insert_value_idx(place, conds, self.map); } @@ -502,21 +504,25 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { box (Operand::Move(place) | Operand::Copy(place), Operand::Constant(value)) | box (Operand::Constant(value), Operand::Move(place) | Operand::Copy(place)), ) => { - let conditions = state.try_get_idx(lhs, self.map)?; - let place = self.map.find(place.as_ref())?; + let Some(conditions) = state.try_get_idx(lhs, self.map) else { return }; + let Some(place) = self.map.find(place.as_ref()) else { return }; let equals = match op { BinOp::Eq => ScalarInt::TRUE, BinOp::Ne => ScalarInt::FALSE, - _ => return None, + _ => return, }; if value.const_.ty().is_floating_point() { // Floating point equality does not follow bit-patterns. // -0.0 and NaN both have special rules for equality, // and therefore we cannot use integer comparisons for them. // Avoid handling them, though this could be extended in the future. - return None; + return; } - let value = value.const_.normalize(self.tcx, self.param_env).try_to_scalar_int()?; + let Some(value) = + value.const_.normalize(self.tcx, self.param_env).try_to_scalar_int() + else { + return; + }; let conds = conditions.map(self.arena, |c| Condition { value, polarity: if c.matches(equals) { Polarity::Eq } else { Polarity::Ne }, @@ -527,8 +533,6 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { _ => {} } - - None } #[instrument(level = "trace", skip(self))] @@ -537,7 +541,7 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { bb: BasicBlock, stmt: &Statement<'tcx>, state: &mut State<ConditionSet<'a>>, - ) -> Option<!> { + ) { let register_opportunity = |c: Condition| { debug!(?bb, ?c.target, "register"); self.opportunities.push(ThreadingOpportunity { chain: vec![bb], target: c.target }) @@ -550,12 +554,12 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { // If we expect `discriminant(place) ?= A`, // we have an opportunity if `variant_index ?= A`. StatementKind::SetDiscriminant { box place, variant_index } => { - let discr_target = self.map.find_discr(place.as_ref())?; + let Some(discr_target) = self.map.find_discr(place.as_ref()) else { return }; let enum_ty = place.ty(self.body, self.tcx).ty; // `SetDiscriminant` may be a no-op if the assigned variant is the untagged variant // of a niche encoding. If we cannot ensure that we write to the discriminant, do // nothing. - let enum_layout = self.ecx.layout_of(enum_ty).ok()?; + let Ok(enum_layout) = self.ecx.layout_of(enum_ty) else { return }; let writes_discriminant = match enum_layout.variants { Variants::Single { index } => { assert_eq!(index, *variant_index); @@ -568,24 +572,25 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { } => *variant_index != untagged_variant, }; if writes_discriminant { - let discr = self.ecx.discriminant_for_variant(enum_ty, *variant_index).ok()?; - self.process_immediate(bb, discr_target, discr, state)?; + let Ok(discr) = self.ecx.discriminant_for_variant(enum_ty, *variant_index) + else { + return; + }; + self.process_immediate(bb, discr_target, discr, state); } } // If we expect `lhs ?= true`, we have an opportunity if we assume `lhs == true`. StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume( Operand::Copy(place) | Operand::Move(place), )) => { - let conditions = state.try_get(place.as_ref(), self.map)?; + let Some(conditions) = state.try_get(place.as_ref(), self.map) else { return }; conditions.iter_matches(ScalarInt::TRUE).for_each(register_opportunity); } StatementKind::Assign(box (lhs_place, rhs)) => { - self.process_assign(bb, lhs_place, rhs, state)?; + self.process_assign(bb, lhs_place, rhs, state); } _ => {} } - - None } #[instrument(level = "trace", skip(self, state, cost))] @@ -638,17 +643,17 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { targets: &SwitchTargets, target_bb: BasicBlock, state: &mut State<ConditionSet<'a>>, - ) -> Option<!> { + ) { debug_assert_ne!(target_bb, START_BLOCK); debug_assert_eq!(self.body.basic_blocks.predecessors()[target_bb].len(), 1); - let discr = discr.place()?; + let Some(discr) = discr.place() else { return }; let discr_ty = discr.ty(self.body, self.tcx).ty; - let discr_layout = self.ecx.layout_of(discr_ty).ok()?; - let conditions = state.try_get(discr.as_ref(), self.map)?; + let Ok(discr_layout) = self.ecx.layout_of(discr_ty) else { return }; + let Some(conditions) = state.try_get(discr.as_ref(), self.map) else { return }; if let Some((value, _)) = targets.iter().find(|&(_, target)| target == target_bb) { - let value = ScalarInt::try_from_uint(value, discr_layout.size)?; + let Some(value) = ScalarInt::try_from_uint(value, discr_layout.size) else { return }; debug_assert_eq!(targets.iter().filter(|&(_, target)| target == target_bb).count(), 1); // We are inside `target_bb`. Since we have a single predecessor, we know we passed @@ -662,7 +667,7 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { } else if let Some((value, _, else_bb)) = targets.as_static_if() && target_bb == else_bb { - let value = ScalarInt::try_from_uint(value, discr_layout.size)?; + let Some(value) = ScalarInt::try_from_uint(value, discr_layout.size) else { return }; // We only know that `discr != value`. That's much weaker information than // the equality we had in the previous arm. All we can conclude is that @@ -675,8 +680,6 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { } } } - - None } } diff --git a/compiler/rustc_mir_transform/src/known_panics_lint.rs b/compiler/rustc_mir_transform/src/known_panics_lint.rs index 7eed47cf239..61405fb25c6 100644 --- a/compiler/rustc_mir_transform/src/known_panics_lint.rs +++ b/compiler/rustc_mir_transform/src/known_panics_lint.rs @@ -22,13 +22,13 @@ use rustc_middle::ty::layout::{LayoutError, LayoutOf, LayoutOfHelpers, TyAndLayo use rustc_middle::ty::{self, ConstInt, ParamEnv, ScalarInt, Ty, TyCtxt, TypeVisitableExt}; use rustc_span::Span; use rustc_target::abi::{Abi, FieldIdx, HasDataLayout, Size, TargetDataLayout, VariantIdx}; +use tracing::{debug, instrument, trace}; use crate::errors::{AssertLint, AssertLintKind}; -use crate::MirLint; pub struct KnownPanicsLint; -impl<'tcx> MirLint<'tcx> for KnownPanicsLint { +impl<'tcx> crate::MirLint<'tcx> for KnownPanicsLint { fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) { if body.tainted_by_errors.is_some() { return; @@ -469,12 +469,12 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { msg: &AssertKind<Operand<'tcx>>, cond: &Operand<'tcx>, location: Location, - ) -> Option<!> { - let value = &self.eval_operand(cond)?; + ) { + let Some(value) = &self.eval_operand(cond) else { return }; trace!("assertion on {:?} should be {:?}", value, expected); let expected = Scalar::from_bool(expected); - let value_const = self.use_ecx(|this| this.ecx.read_scalar(value))?; + let Some(value_const) = self.use_ecx(|this| this.ecx.read_scalar(value)) else { return }; if expected != value_const { // Poison all places this operand references so that further code @@ -516,14 +516,12 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { AssertKind::BoundsCheck { len, index } } // Remaining overflow errors are already covered by checks on the binary operators. - AssertKind::Overflow(..) | AssertKind::OverflowNeg(_) => return None, + AssertKind::Overflow(..) | AssertKind::OverflowNeg(_) => return, // Need proper const propagator for these. - _ => return None, + _ => return, }; self.report_assert_as_lint(location, AssertLintKind::UnconditionalPanic, msg); } - - None } fn ensure_not_propagated(&self, local: Local) { @@ -915,7 +913,7 @@ impl<'tcx> Visitor<'tcx> for CanConstProp { fn visit_place(&mut self, place: &Place<'tcx>, mut context: PlaceContext, loc: Location) { use rustc_middle::mir::visit::PlaceContext::*; - // Dereferencing just read the addess of `place.local`. + // Dereferencing just read the address of `place.local`. if place.projection.first() == Some(&PlaceElem::Deref) { context = NonMutatingUse(NonMutatingUseContext::Copy); } diff --git a/compiler/rustc_mir_transform/src/large_enums.rs b/compiler/rustc_mir_transform/src/large_enums.rs index cbc3169f2f1..f02ba71ddc6 100644 --- a/compiler/rustc_mir_transform/src/large_enums.rs +++ b/compiler/rustc_mir_transform/src/large_enums.rs @@ -27,7 +27,7 @@ pub struct EnumSizeOpt { pub(crate) discrepancy: u64, } -impl<'tcx> MirPass<'tcx> for EnumSizeOpt { +impl<'tcx> crate::MirPass<'tcx> for EnumSizeOpt { fn is_enabled(&self, sess: &Session) -> bool { // There are some differences in behavior on wasm and ARM that are not properly // understood, so we conservatively treat this optimization as unsound: diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 1f214bc42cb..62e73ba2c8e 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -3,23 +3,19 @@ #![feature(box_patterns)] #![feature(const_type_name)] #![feature(cow_is_borrowed)] -#![feature(decl_macro)] #![feature(if_let_guard)] #![feature(impl_trait_in_assoc_type)] #![feature(let_chains)] #![feature(map_try_insert)] #![feature(never_type)] -#![feature(option_get_or_insert_default)] #![feature(round_char_boundary)] #![feature(try_blocks)] #![feature(yeet_expr)] // tidy-alphabetical-end -#[macro_use] -extern crate tracing; - use hir::ConstContext; use required_consts::RequiredConstsVisitor; +use rustc_const_eval::check_consts::{self, ConstCx}; use rustc_const_eval::util; use rustc_data_structures::fx::FxIndexSet; use rustc_data_structures::steal::Steal; @@ -30,7 +26,7 @@ use rustc_hir::intravisit::{self, Visitor}; use rustc_index::IndexVec; use rustc_middle::mir::{ AnalysisPhase, Body, CallSource, ClearCrossCrate, ConstOperand, ConstQualifs, LocalDecl, - MirPass, MirPhase, Operand, Place, ProjectionElem, Promoted, RuntimePhase, Rvalue, SourceInfo, + MirPhase, Operand, Place, ProjectionElem, Promoted, RuntimePhase, Rvalue, SourceInfo, Statement, StatementKind, TerminatorKind, START_BLOCK, }; use rustc_middle::ty::{self, TyCtxt, TypeVisitableExt}; @@ -39,11 +35,12 @@ use rustc_middle::{bug, query, span_bug}; use rustc_span::source_map::Spanned; use rustc_span::{sym, DUMMY_SP}; use rustc_trait_selection::traits; +use tracing::{debug, trace}; #[macro_use] mod pass_manager; -use pass_manager::{self as pm, Lint, MirLint, WithMinOptLevel}; +use pass_manager::{self as pm, Lint, MirLint, MirPass, WithMinOptLevel}; mod abort_unwinding_calls; mod add_call_guards; @@ -98,6 +95,7 @@ mod remove_unneeded_drops; mod remove_zsts; mod required_consts; mod reveal_all; +mod sanity_check; mod shim; mod ssa; // This pass is public to allow external drivers to perform MIR cleanup @@ -110,9 +108,6 @@ mod unreachable_enum_branching; mod unreachable_prop; mod validate; -use rustc_const_eval::check_consts::{self, ConstCx}; -use rustc_mir_dataflow::rustc_peek; - rustc_fluent_macro::fluent_messages! { "../messages.ftl" } pub fn provide(providers: &mut Providers) { @@ -130,11 +125,12 @@ pub fn provide(providers: &mut Providers) { mir_coroutine_witnesses: coroutine::mir_coroutine_witnesses, optimized_mir, is_mir_available, - is_ctfe_mir_available: |tcx, did| is_mir_available(tcx, did), + is_ctfe_mir_available: is_mir_available, mir_callgraph_reachable: inline::cycle::mir_callgraph_reachable, mir_inliner_callees: inline::cycle::mir_inliner_callees, promoted_mir, deduced_param_attrs: deduce_param_attrs::deduced_param_attrs, + coroutine_by_move_body_def_id: coroutine::coroutine_by_move_body_def_id, ..providers.queries }; } @@ -222,10 +218,8 @@ fn is_mir_available(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { /// Finds the full set of `DefId`s within the current crate that have /// MIR associated with them. fn mir_keys(tcx: TyCtxt<'_>, (): ()) -> FxIndexSet<LocalDefId> { - let mut set = FxIndexSet::default(); - // All body-owners have MIR associated with them. - set.extend(tcx.hir().body_owners()); + let mut set: FxIndexSet<_> = tcx.hir().body_owners().collect(); // Additionally, tuple struct/variant constructors have MIR, but // they don't have a BodyId, so we need to build them separately. @@ -250,8 +244,7 @@ fn mir_const_qualif(tcx: TyCtxt<'_>, def: LocalDefId) -> ConstQualifs { // No need to const-check a non-const `fn`. match const_kind { - Some(ConstContext::Const { .. } | ConstContext::Static(_)) - | Some(ConstContext::ConstFn) => {} + Some(ConstContext::Const { .. } | ConstContext::Static(_) | ConstContext::ConstFn) => {} None => span_bug!( tcx.def_span(def), "`mir_const_qualif` should only be called on const fns and const items" @@ -293,13 +286,9 @@ fn mir_built(tcx: TyCtxt<'_>, def: LocalDefId) -> &Steal<Body<'_>> { &Lint(check_packed_ref::CheckPackedRef), &Lint(check_const_item_mutation::CheckConstItemMutation), &Lint(function_item_references::FunctionItemReferences), - // If this is an async closure's output coroutine, generate - // by-move and by-mut bodies if needed. We do this first so - // they can be optimized in lockstep with their parent bodies. - &coroutine::ByMoveBody, // What we need to do constant evaluation. &simplify::SimplifyCfg::Initial, - &rustc_peek::SanityCheck, // Just a lint + &Lint(sanity_check::SanityCheck), ], None, ); @@ -329,8 +318,15 @@ fn mir_promoted( | DefKind::AnonConst => tcx.mir_const_qualif(def), _ => ConstQualifs::default(), }; - // has_ffi_unwind_calls query uses the raw mir, so make sure it is run. + + // the `has_ffi_unwind_calls` query uses the raw mir, so make sure it is run. tcx.ensure_with_value().has_ffi_unwind_calls(def); + + // the `by_move_body` query uses the raw mir, so make sure it is run. + if tcx.needs_coroutine_by_move_body_def_id(def) { + tcx.ensure_with_value().coroutine_by_move_body_def_id(def); + } + let mut body = tcx.mir_built(def).steal(); if let Some(error_reported) = const_qualifs.tainted_by_errors { body.tainted_by_errors = Some(error_reported); @@ -339,14 +335,6 @@ fn mir_promoted( // Collect `required_consts` *before* promotion, so if there are any consts being promoted // we still add them to the list in the outer MIR body. RequiredConstsVisitor::compute_required_consts(&mut body); - // If this has an associated by-move async closure body, that doesn't get run through these - // passes itself, it gets "tagged along" by the pass manager. `RequiredConstsVisitor` is not - // a regular pass so we have to also apply it manually to the other body. - if let Some(coroutine) = body.coroutine.as_mut() { - if let Some(by_move_body) = coroutine.by_move_body.as_mut() { - RequiredConstsVisitor::compute_required_consts(by_move_body); - } - } // What we need to run borrowck etc. let promote_pass = promote_consts::PromoteTemps::default(); @@ -398,7 +386,10 @@ fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> & if tcx.is_coroutine(def.to_def_id()) { tcx.ensure_with_value().mir_coroutine_witnesses(def); } - let mir_borrowck = tcx.mir_borrowck(def); + + // We only need to borrowck non-synthetic MIR. + let tainted_by_errors = + if !tcx.is_synthetic_mir(def) { tcx.mir_borrowck(def).tainted_by_errors } else { None }; let is_fn_like = tcx.def_kind(def).is_fn_like(); if is_fn_like { @@ -410,7 +401,8 @@ fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> & let (body, _) = tcx.mir_promoted(def); let mut body = body.steal(); - if let Some(error_reported) = mir_borrowck.tainted_by_errors { + + if let Some(error_reported) = tainted_by_errors { body.tainted_by_errors = Some(error_reported); } @@ -660,14 +652,6 @@ fn inner_optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> Body<'_> { // visited does not depend on the optimization level. // We do not use `run_passes` for this as that might skip the pass if `injection_phase` is set. mentioned_items::MentionedItems.run_pass(tcx, &mut body); - // If this has an associated by-move async closure body, that doesn't get run through these - // passes itself, it gets "tagged along" by the pass manager. Since we're not using the pass - // manager we have to do this by hand. - if let Some(coroutine) = body.coroutine.as_mut() { - if let Some(by_move_body) = coroutine.by_move_body.as_mut() { - mentioned_items::MentionedItems.run_pass(tcx, by_move_body); - } - } // If `mir_drops_elaborated_and_const_checked` found that the current body has unsatisfiable // predicates, it will shrink the MIR to a single `unreachable` terminator. @@ -690,7 +674,9 @@ fn promoted_mir(tcx: TyCtxt<'_>, def: LocalDefId) -> &IndexVec<Promoted, Body<'_ return tcx.arena.alloc(IndexVec::new()); } - tcx.ensure_with_value().mir_borrowck(def); + if !tcx.is_synthetic_mir(def) { + tcx.ensure_with_value().mir_borrowck(def); + } let mut promoted = tcx.mir_promoted(def).1.steal(); for body in &mut promoted { diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs index a9bdff95fe5..55eec332306 100644 --- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs +++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs @@ -9,7 +9,7 @@ use crate::take_array; pub struct LowerIntrinsics; -impl<'tcx> MirPass<'tcx> for LowerIntrinsics { +impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let local_decls = &body.local_decls; for block in body.basic_blocks.as_mut() { diff --git a/compiler/rustc_mir_transform/src/lower_slice_len.rs b/compiler/rustc_mir_transform/src/lower_slice_len.rs index 77a7f4f47dd..555309a7750 100644 --- a/compiler/rustc_mir_transform/src/lower_slice_len.rs +++ b/compiler/rustc_mir_transform/src/lower_slice_len.rs @@ -7,7 +7,7 @@ use rustc_middle::ty::TyCtxt; pub struct LowerSliceLenCalls; -impl<'tcx> MirPass<'tcx> for LowerSliceLenCalls { +impl<'tcx> crate::MirPass<'tcx> for LowerSliceLenCalls { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { sess.mir_opt_level() > 0 } diff --git a/compiler/rustc_mir_transform/src/match_branches.rs b/compiler/rustc_mir_transform/src/match_branches.rs index 47758b56f8c..233b39fb47a 100644 --- a/compiler/rustc_mir_transform/src/match_branches.rs +++ b/compiler/rustc_mir_transform/src/match_branches.rs @@ -12,7 +12,7 @@ use super::simplify::simplify_cfg; pub struct MatchBranchSimplification; -impl<'tcx> MirPass<'tcx> for MatchBranchSimplification { +impl<'tcx> crate::MirPass<'tcx> for MatchBranchSimplification { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { sess.mir_opt_level() >= 1 } @@ -289,7 +289,7 @@ fn can_cast( #[derive(Default)] struct SimplifyToExp { - transfrom_kinds: Vec<TransfromKind>, + transform_kinds: Vec<TransformKind>, } #[derive(Clone, Copy)] @@ -302,17 +302,17 @@ enum ExpectedTransformKind<'tcx, 'a> { Cast { place: &'a Place<'tcx>, ty: Ty<'tcx> }, } -enum TransfromKind { +enum TransformKind { Same, Cast, } -impl From<ExpectedTransformKind<'_, '_>> for TransfromKind { +impl From<ExpectedTransformKind<'_, '_>> for TransformKind { fn from(compare_type: ExpectedTransformKind<'_, '_>) -> Self { match compare_type { - ExpectedTransformKind::Same(_) => TransfromKind::Same, - ExpectedTransformKind::SameByEq { .. } => TransfromKind::Same, - ExpectedTransformKind::Cast { .. } => TransfromKind::Cast, + ExpectedTransformKind::Same(_) => TransformKind::Same, + ExpectedTransformKind::SameByEq { .. } => TransformKind::Same, + ExpectedTransformKind::Cast { .. } => TransformKind::Cast, } } } @@ -475,7 +475,7 @@ impl<'tcx> SimplifyMatch<'tcx> for SimplifyToExp { } } } - self.transfrom_kinds = expected_transform_kinds.into_iter().map(|c| c.into()).collect(); + self.transform_kinds = expected_transform_kinds.into_iter().map(|c| c.into()).collect(); Some(()) } @@ -493,13 +493,13 @@ impl<'tcx> SimplifyMatch<'tcx> for SimplifyToExp { let (_, first) = targets.iter().next().unwrap(); let first = &bbs[first]; - for (t, s) in iter::zip(&self.transfrom_kinds, &first.statements) { + for (t, s) in iter::zip(&self.transform_kinds, &first.statements) { match (t, &s.kind) { - (TransfromKind::Same, _) => { + (TransformKind::Same, _) => { patch.add_statement(parent_end, s.kind.clone()); } ( - TransfromKind::Cast, + TransformKind::Cast, StatementKind::Assign(box (lhs, Rvalue::Use(Operand::Constant(f_c)))), ) => { let operand = Operand::Copy(Place::from(discr_local)); diff --git a/compiler/rustc_mir_transform/src/mentioned_items.rs b/compiler/rustc_mir_transform/src/mentioned_items.rs index 32c8064ebca..41ce03caf08 100644 --- a/compiler/rustc_mir_transform/src/mentioned_items.rs +++ b/compiler/rustc_mir_transform/src/mentioned_items.rs @@ -1,5 +1,5 @@ use rustc_middle::mir::visit::Visitor; -use rustc_middle::mir::{self, Location, MentionedItem, MirPass}; +use rustc_middle::mir::{self, Location, MentionedItem}; use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::{self, TyCtxt}; use rustc_session::Session; @@ -13,7 +13,7 @@ struct MentionedItemsVisitor<'a, 'tcx> { mentioned_items: &'a mut Vec<Spanned<MentionedItem<'tcx>>>, } -impl<'tcx> MirPass<'tcx> for MentionedItems { +impl<'tcx> crate::MirPass<'tcx> for MentionedItems { fn is_enabled(&self, _sess: &Session) -> bool { // If this pass is skipped the collector assume that nothing got mentioned! We could // potentially skip it in opt-level 0 if we are sure that opt-level will never *remove* uses diff --git a/compiler/rustc_mir_transform/src/multiple_return_terminators.rs b/compiler/rustc_mir_transform/src/multiple_return_terminators.rs index 1e87a0e01d9..1b4972d487e 100644 --- a/compiler/rustc_mir_transform/src/multiple_return_terminators.rs +++ b/compiler/rustc_mir_transform/src/multiple_return_terminators.rs @@ -9,7 +9,7 @@ use crate::simplify; pub struct MultipleReturnTerminators; -impl<'tcx> MirPass<'tcx> for MultipleReturnTerminators { +impl<'tcx> crate::MirPass<'tcx> for MultipleReturnTerminators { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { sess.mir_opt_level() >= 4 } diff --git a/compiler/rustc_mir_transform/src/nrvo.rs b/compiler/rustc_mir_transform/src/nrvo.rs index 885dbd5f339..94573a9d89b 100644 --- a/compiler/rustc_mir_transform/src/nrvo.rs +++ b/compiler/rustc_mir_transform/src/nrvo.rs @@ -6,8 +6,7 @@ use rustc_middle::bug; use rustc_middle::mir::visit::{MutVisitor, NonUseContext, PlaceContext, Visitor}; use rustc_middle::mir::{self, BasicBlock, Local, Location}; use rustc_middle::ty::TyCtxt; - -use crate::MirPass; +use tracing::{debug, trace}; /// This pass looks for MIR that always copies the same local into the return place and eliminates /// the copy by renaming all uses of that local to `_0`. @@ -33,7 +32,7 @@ use crate::MirPass; /// [#71003]: https://github.com/rust-lang/rust/pull/71003 pub struct RenameReturnPlace; -impl<'tcx> MirPass<'tcx> for RenameReturnPlace { +impl<'tcx> crate::MirPass<'tcx> for RenameReturnPlace { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { // unsound: #111005 sess.mir_opt_level() > 0 && sess.opts.unstable_opts.unsound_mir_opts diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs index 824a4b2f2df..28d4e1a1c91 100644 --- a/compiler/rustc_mir_transform/src/pass_manager.rs +++ b/compiler/rustc_mir_transform/src/pass_manager.rs @@ -1,18 +1,99 @@ +use std::cell::RefCell; +use std::collections::hash_map::Entry; + +use rustc_data_structures::fx::FxHashMap; use rustc_middle::mir::{self, Body, MirPhase, RuntimePhase}; use rustc_middle::ty::TyCtxt; use rustc_session::Session; +use tracing::trace; use crate::lint::lint_body; -use crate::{validate, MirPass}; +use crate::validate; + +thread_local! { + static PASS_NAMES: RefCell<FxHashMap<&'static str, &'static str>> = { + RefCell::new(FxHashMap::default()) + }; +} + +/// Converts a MIR pass name into a snake case form to match the profiling naming style. +fn to_profiler_name(type_name: &'static str) -> &'static str { + PASS_NAMES.with(|names| match names.borrow_mut().entry(type_name) { + Entry::Occupied(e) => *e.get(), + Entry::Vacant(e) => { + let snake_case: String = type_name + .chars() + .flat_map(|c| { + if c.is_ascii_uppercase() { + vec!['_', c.to_ascii_lowercase()] + } else if c == '-' { + vec!['_'] + } else { + vec![c] + } + }) + .collect(); + let result = &*String::leak(format!("mir_pass{}", snake_case)); + e.insert(result); + result + } + }) +} + +// const wrapper for `if let Some((_, tail)) = name.rsplit_once(':') { tail } else { name }` +const fn c_name(name: &'static str) -> &'static str { + // FIXME Simplify the implementation once more `str` methods get const-stable. + // and inline into call site + let bytes = name.as_bytes(); + let mut i = bytes.len(); + while i > 0 && bytes[i - 1] != b':' { + i = i - 1; + } + let (_, bytes) = bytes.split_at(i); + match std::str::from_utf8(bytes) { + Ok(name) => name, + Err(_) => name, + } +} + +/// A streamlined trait that you can implement to create a pass; the +/// pass will be named after the type, and it will consist of a main +/// loop that goes over each available MIR and applies `run_pass`. +pub(super) trait MirPass<'tcx> { + fn name(&self) -> &'static str { + // FIXME Simplify the implementation once more `str` methods get const-stable. + // See copypaste in `MirLint` + const { + let name = std::any::type_name::<Self>(); + c_name(name) + } + } + + fn profiler_name(&self) -> &'static str { + to_profiler_name(self.name()) + } + + /// Returns `true` if this pass is enabled with the current combination of compiler flags. + fn is_enabled(&self, _sess: &Session) -> bool { + true + } + + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>); -/// Just like `MirPass`, except it cannot mutate `Body`. -pub trait MirLint<'tcx> { + fn is_mir_dump_enabled(&self) -> bool { + true + } +} + +/// Just like `MirPass`, except it cannot mutate `Body`, and MIR dumping is +/// disabled (via the `Lint` adapter). +pub(super) trait MirLint<'tcx> { fn name(&self) -> &'static str { // FIXME Simplify the implementation once more `str` methods get const-stable. // See copypaste in `MirPass` const { let name = std::any::type_name::<Self>(); - rustc_middle::util::common::c_name(name) + c_name(name) } } @@ -25,7 +106,7 @@ pub trait MirLint<'tcx> { /// An adapter for `MirLint`s that implements `MirPass`. #[derive(Debug, Clone)] -pub struct Lint<T>(pub T); +pub(super) struct Lint<T>(pub T); impl<'tcx, T> MirPass<'tcx> for Lint<T> where @@ -48,7 +129,7 @@ where } } -pub struct WithMinOptLevel<T>(pub u32, pub T); +pub(super) struct WithMinOptLevel<T>(pub u32, pub T); impl<'tcx, T> MirPass<'tcx> for WithMinOptLevel<T> where @@ -69,7 +150,7 @@ where /// Run the sequence of passes without validating the MIR after each pass. The MIR is still /// validated at the end. -pub fn run_passes_no_validate<'tcx>( +pub(super) fn run_passes_no_validate<'tcx>( tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, passes: &[&dyn MirPass<'tcx>], @@ -79,7 +160,7 @@ pub fn run_passes_no_validate<'tcx>( } /// The optional `phase_change` is applied after executing all the passes, if present -pub fn run_passes<'tcx>( +pub(super) fn run_passes<'tcx>( tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, passes: &[&dyn MirPass<'tcx>], @@ -88,7 +169,7 @@ pub fn run_passes<'tcx>( run_passes_inner(tcx, body, passes, phase_change, true); } -pub fn should_run_pass<'tcx, P>(tcx: TyCtxt<'tcx>, pass: &P) -> bool +pub(super) fn should_run_pass<'tcx, P>(tcx: TyCtxt<'tcx>, pass: &P) -> bool where P: MirPass<'tcx> + ?Sized, { @@ -182,19 +263,13 @@ fn run_passes_inner<'tcx>( body.pass_count = 1; } - - if let Some(coroutine) = body.coroutine.as_mut() { - if let Some(by_move_body) = coroutine.by_move_body.as_mut() { - run_passes_inner(tcx, by_move_body, passes, phase_change, validate_each); - } - } } -pub fn validate_body<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, when: String) { +pub(super) fn validate_body<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, when: String) { validate::Validator { when, mir_phase: body.phase }.run_pass(tcx, body); } -pub fn dump_mir_for_pass<'tcx>( +pub(super) fn dump_mir_for_pass<'tcx>( tcx: TyCtxt<'tcx>, body: &Body<'tcx>, pass_name: &str, @@ -210,7 +285,7 @@ pub fn dump_mir_for_pass<'tcx>( ); } -pub fn dump_mir_for_phase_change<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) { +pub(super) fn dump_mir_for_phase_change<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) { assert_eq!(body.pass_count, 0); mir::dump_mir(tcx, true, body.phase.name(), &"after", body, |_, _| Ok(())) } diff --git a/compiler/rustc_mir_transform/src/prettify.rs b/compiler/rustc_mir_transform/src/prettify.rs index 14dd0c6f61e..ad71c622660 100644 --- a/compiler/rustc_mir_transform/src/prettify.rs +++ b/compiler/rustc_mir_transform/src/prettify.rs @@ -17,7 +17,7 @@ use rustc_session::Session; /// `IndexVec`, unless that successor is a back-edge (such as from a loop). pub struct ReorderBasicBlocks; -impl<'tcx> MirPass<'tcx> for ReorderBasicBlocks { +impl<'tcx> crate::MirPass<'tcx> for ReorderBasicBlocks { fn is_enabled(&self, _session: &Session) -> bool { false } @@ -45,7 +45,7 @@ impl<'tcx> MirPass<'tcx> for ReorderBasicBlocks { /// (Does not reorder arguments nor the [`RETURN_PLACE`].) pub struct ReorderLocals; -impl<'tcx> MirPass<'tcx> for ReorderLocals { +impl<'tcx> crate::MirPass<'tcx> for ReorderLocals { fn is_enabled(&self, _session: &Session) -> bool { false } diff --git a/compiler/rustc_mir_transform/src/promote_consts.rs b/compiler/rustc_mir_transform/src/promote_consts.rs index 6e84914ef97..cf0a569ffa4 100644 --- a/compiler/rustc_mir_transform/src/promote_consts.rs +++ b/compiler/rustc_mir_transform/src/promote_consts.rs @@ -27,6 +27,7 @@ use rustc_middle::ty::{self, GenericArgs, List, Ty, TyCtxt, TypeVisitableExt}; use rustc_middle::{bug, mir, span_bug}; use rustc_span::source_map::Spanned; use rustc_span::Span; +use tracing::{debug, instrument}; /// A `MirPass` for promotion. /// @@ -40,7 +41,7 @@ pub struct PromoteTemps<'tcx> { pub promoted_fragments: Cell<IndexVec<Promoted, Body<'tcx>>>, } -impl<'tcx> MirPass<'tcx> for PromoteTemps<'tcx> { +impl<'tcx> crate::MirPass<'tcx> for PromoteTemps<'tcx> { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // There's not really any point in promoting errorful MIR. // @@ -497,7 +498,7 @@ impl<'tcx> Validator<'_, 'tcx> { Some(x) if x != 0 => {} // okay _ => return Err(Unpromotable), // value not known or 0 -- not okay } - // Furthermore, for signed divison, we also have to exclude `int::MIN / -1`. + // Furthermore, for signed division, we also have to exclude `int::MIN / -1`. if lhs_ty.is_signed() { match rhs_val.map(|x| x.to_int(sz)) { Some(-1) | None => { diff --git a/compiler/rustc_mir_transform/src/ref_prop.rs b/compiler/rustc_mir_transform/src/ref_prop.rs index 973a191d786..4a447d24cce 100644 --- a/compiler/rustc_mir_transform/src/ref_prop.rs +++ b/compiler/rustc_mir_transform/src/ref_prop.rs @@ -10,6 +10,7 @@ use rustc_middle::ty::TyCtxt; use rustc_mir_dataflow::impls::MaybeStorageDead; use rustc_mir_dataflow::storage::always_storage_live_locals; use rustc_mir_dataflow::Analysis; +use tracing::{debug, instrument}; use crate::ssa::{SsaLocals, StorageLiveLocals}; @@ -71,7 +72,7 @@ use crate::ssa::{SsaLocals, StorageLiveLocals}; /// so we perform all the possible instantiations without removing the `_1 = &_2` statement. pub struct ReferencePropagation; -impl<'tcx> MirPass<'tcx> for ReferencePropagation { +impl<'tcx> crate::MirPass<'tcx> for ReferencePropagation { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { sess.mir_opt_level() >= 2 } @@ -344,7 +345,7 @@ fn fully_replacable_locals(ssa: &SsaLocals) -> BitSet<Local> { replacable } -/// Utility to help performing subtitution of `*pattern` by `target`. +/// Utility to help performing substitution of `*pattern` by `target`. struct Replacer<'tcx> { tcx: TyCtxt<'tcx>, targets: IndexVec<Local, Value<'tcx>>, diff --git a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs index 1df5737e859..ccba8d015e3 100644 --- a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs +++ b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs @@ -3,13 +3,14 @@ use rustc_middle::mir::patch::MirPatch; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; use rustc_target::spec::PanicStrategy; +use tracing::debug; /// A pass that removes noop landing pads and replaces jumps to them with /// `UnwindAction::Continue`. This is important because otherwise LLVM generates /// terrible code for these. pub struct RemoveNoopLandingPads; -impl<'tcx> MirPass<'tcx> for RemoveNoopLandingPads { +impl<'tcx> crate::MirPass<'tcx> for RemoveNoopLandingPads { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { sess.panic_strategy() != PanicStrategy::Abort } diff --git a/compiler/rustc_mir_transform/src/remove_place_mention.rs b/compiler/rustc_mir_transform/src/remove_place_mention.rs index 78335b3b5e0..5801fdedceb 100644 --- a/compiler/rustc_mir_transform/src/remove_place_mention.rs +++ b/compiler/rustc_mir_transform/src/remove_place_mention.rs @@ -2,10 +2,11 @@ use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; +use tracing::trace; pub struct RemovePlaceMention; -impl<'tcx> MirPass<'tcx> for RemovePlaceMention { +impl<'tcx> crate::MirPass<'tcx> for RemovePlaceMention { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { !sess.opts.unstable_opts.mir_keep_place_mention } diff --git a/compiler/rustc_mir_transform/src/remove_storage_markers.rs b/compiler/rustc_mir_transform/src/remove_storage_markers.rs index f68e592db15..329b30d3890 100644 --- a/compiler/rustc_mir_transform/src/remove_storage_markers.rs +++ b/compiler/rustc_mir_transform/src/remove_storage_markers.rs @@ -2,10 +2,11 @@ use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; +use tracing::trace; pub struct RemoveStorageMarkers; -impl<'tcx> MirPass<'tcx> for RemoveStorageMarkers { +impl<'tcx> crate::MirPass<'tcx> for RemoveStorageMarkers { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { sess.mir_opt_level() > 0 && !sess.emit_lifetime_markers() } diff --git a/compiler/rustc_mir_transform/src/remove_uninit_drops.rs b/compiler/rustc_mir_transform/src/remove_uninit_drops.rs index fae1cb5f7d8..aafe971311d 100644 --- a/compiler/rustc_mir_transform/src/remove_uninit_drops.rs +++ b/compiler/rustc_mir_transform/src/remove_uninit_drops.rs @@ -6,8 +6,6 @@ use rustc_mir_dataflow::move_paths::{LookupResult, MoveData, MovePathIndex}; use rustc_mir_dataflow::{move_path_children_matching, Analysis, MaybeReachable}; use rustc_target::abi::FieldIdx; -use crate::MirPass; - /// Removes `Drop` terminators whose target is known to be uninitialized at /// that point. /// @@ -18,7 +16,7 @@ use crate::MirPass; /// [#90770]: https://github.com/rust-lang/rust/issues/90770 pub struct RemoveUninitDrops; -impl<'tcx> MirPass<'tcx> for RemoveUninitDrops { +impl<'tcx> crate::MirPass<'tcx> for RemoveUninitDrops { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let param_env = tcx.param_env(body.source.def_id()); let move_data = diff --git a/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs b/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs index 2778d91e17b..43109aae0fb 100644 --- a/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs +++ b/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs @@ -6,12 +6,13 @@ use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; +use tracing::{debug, trace}; use super::simplify::simplify_cfg; pub struct RemoveUnneededDrops; -impl<'tcx> MirPass<'tcx> for RemoveUnneededDrops { +impl<'tcx> crate::MirPass<'tcx> for RemoveUnneededDrops { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { trace!("Running RemoveUnneededDrops on {:?}", body.source); diff --git a/compiler/rustc_mir_transform/src/remove_zsts.rs b/compiler/rustc_mir_transform/src/remove_zsts.rs index 9a94cae3382..9aa46bd4fba 100644 --- a/compiler/rustc_mir_transform/src/remove_zsts.rs +++ b/compiler/rustc_mir_transform/src/remove_zsts.rs @@ -6,7 +6,7 @@ use rustc_middle::ty::{self, Ty, TyCtxt}; pub struct RemoveZsts; -impl<'tcx> MirPass<'tcx> for RemoveZsts { +impl<'tcx> crate::MirPass<'tcx> for RemoveZsts { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { sess.mir_opt_level() > 0 } diff --git a/compiler/rustc_mir_transform/src/reveal_all.rs b/compiler/rustc_mir_transform/src/reveal_all.rs index 5eaa024f846..29312a99cbc 100644 --- a/compiler/rustc_mir_transform/src/reveal_all.rs +++ b/compiler/rustc_mir_transform/src/reveal_all.rs @@ -6,7 +6,7 @@ use rustc_middle::ty::{self, Ty, TyCtxt}; pub struct RevealAll; -impl<'tcx> MirPass<'tcx> for RevealAll { +impl<'tcx> crate::MirPass<'tcx> for RevealAll { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); RevealAllVisitor { tcx, param_env }.visit_body_preserves_cfg(body); diff --git a/compiler/rustc_mir_transform/src/sanity_check.rs b/compiler/rustc_mir_transform/src/sanity_check.rs new file mode 100644 index 00000000000..c9445d18162 --- /dev/null +++ b/compiler/rustc_mir_transform/src/sanity_check.rs @@ -0,0 +1,11 @@ +use rustc_middle::mir::Body; +use rustc_middle::ty::TyCtxt; +use rustc_mir_dataflow::rustc_peek::sanity_check; + +pub(super) struct SanityCheck; + +impl<'tcx> crate::MirLint<'tcx> for SanityCheck { + fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) { + sanity_check(tcx, body); + } +} diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index 09b4e5e0711..8c70e429118 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -17,6 +17,7 @@ use rustc_span::source_map::Spanned; use rustc_span::{Span, DUMMY_SP}; use rustc_target::abi::{FieldIdx, VariantIdx, FIRST_VARIANT}; use rustc_target::spec::abi::Abi; +use tracing::{debug, instrument}; use crate::{ abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, deref_separator, @@ -78,15 +79,11 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body< receiver_by_ref, } => build_construct_coroutine_by_move_shim(tcx, coroutine_closure_def_id, receiver_by_ref), - ty::InstanceKind::CoroutineKindShim { coroutine_def_id } => { - return tcx.optimized_mir(coroutine_def_id).coroutine_by_move_body().unwrap().clone(); - } - ty::InstanceKind::DropGlue(def_id, ty) => { // FIXME(#91576): Drop shims for coroutines aren't subject to the MIR passes at the end // of this function. Is this intentional? - if let Some(ty::Coroutine(coroutine_def_id, args)) = ty.map(Ty::kind) { - let coroutine_body = tcx.optimized_mir(*coroutine_def_id); + if let Some(&ty::Coroutine(coroutine_def_id, args)) = ty.map(Ty::kind) { + let coroutine_body = tcx.optimized_mir(coroutine_def_id); let ty::Coroutine(_, id_args) = *tcx.type_of(coroutine_def_id).skip_binder().kind() else { @@ -105,7 +102,9 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceKind<'tcx>) -> Body< args.as_coroutine().kind_ty().to_opt_closure_kind().unwrap(), ty::ClosureKind::FnOnce ); - coroutine_body.coroutine_by_move_body().unwrap().coroutine_drop().unwrap() + tcx.optimized_mir(tcx.coroutine_by_move_body_def_id(coroutine_def_id)) + .coroutine_drop() + .unwrap() }; let mut body = EarlyBinder::bind(body.clone()).instantiate(tcx, args); diff --git a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs index 9c3f903e0ea..50810f23113 100644 --- a/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs +++ b/compiler/rustc_mir_transform/src/shim/async_destructor_ctor.rs @@ -19,6 +19,7 @@ use rustc_span::source_map::respan; use rustc_span::{Span, Symbol}; use rustc_target::abi::{FieldIdx, VariantIdx}; use rustc_target::spec::PanicStrategy; +use tracing::debug; use super::{local_decls_for_sig, new_body}; diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index 4fe8cf6213f..1478b86d3c7 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -33,6 +33,7 @@ use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; use rustc_span::DUMMY_SP; use smallvec::SmallVec; +use tracing::{debug, trace}; pub enum SimplifyCfg { Initial, @@ -73,7 +74,7 @@ pub(crate) fn simplify_cfg(body: &mut Body<'_>) { body.basic_blocks_mut().raw.shrink_to_fit(); } -impl<'tcx> MirPass<'tcx> for SimplifyCfg { +impl<'tcx> crate::MirPass<'tcx> for SimplifyCfg { fn name(&self) -> &'static str { self.name() } @@ -365,7 +366,7 @@ pub enum SimplifyLocals { Final, } -impl<'tcx> MirPass<'tcx> for SimplifyLocals { +impl<'tcx> crate::MirPass<'tcx> for SimplifyLocals { fn name(&self) -> &'static str { match &self { SimplifyLocals::BeforeConstProp => "SimplifyLocals-before-const-prop", diff --git a/compiler/rustc_mir_transform/src/simplify_branches.rs b/compiler/rustc_mir_transform/src/simplify_branches.rs index c746041ebd8..5a014bb7346 100644 --- a/compiler/rustc_mir_transform/src/simplify_branches.rs +++ b/compiler/rustc_mir_transform/src/simplify_branches.rs @@ -1,12 +1,13 @@ use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; +use tracing::trace; pub enum SimplifyConstCondition { AfterConstProp, Final, } /// A pass that replaces a branch with a goto when its condition is known. -impl<'tcx> MirPass<'tcx> for SimplifyConstCondition { +impl<'tcx> crate::MirPass<'tcx> for SimplifyConstCondition { fn name(&self) -> &'static str { match self { SimplifyConstCondition::AfterConstProp => "SimplifyConstCondition-after-const-prop", diff --git a/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs b/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs index 59f67d8e73f..bd30ecc59b3 100644 --- a/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs +++ b/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs @@ -7,8 +7,7 @@ use rustc_middle::mir::{ TerminatorKind, }; use rustc_middle::ty::{Ty, TyCtxt}; - -use super::MirPass; +use tracing::trace; /// Pass to convert `if` conditions on integrals into switches on the integral. /// For an example, it turns something like @@ -26,7 +25,7 @@ use super::MirPass; /// ``` pub struct SimplifyComparisonIntegral; -impl<'tcx> MirPass<'tcx> for SimplifyComparisonIntegral { +impl<'tcx> crate::MirPass<'tcx> for SimplifyComparisonIntegral { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { sess.mir_opt_level() > 0 } diff --git a/compiler/rustc_mir_transform/src/single_use_consts.rs b/compiler/rustc_mir_transform/src/single_use_consts.rs index 35cb6872fe9..64a92872830 100644 --- a/compiler/rustc_mir_transform/src/single_use_consts.rs +++ b/compiler/rustc_mir_transform/src/single_use_consts.rs @@ -21,7 +21,7 @@ use rustc_middle::ty::TyCtxt; /// needed to do that too, including updating the debug info. pub struct SingleUseConsts; -impl<'tcx> MirPass<'tcx> for SingleUseConsts { +impl<'tcx> crate::MirPass<'tcx> for SingleUseConsts { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { sess.mir_opt_level() > 0 } diff --git a/compiler/rustc_mir_transform/src/sroa.rs b/compiler/rustc_mir_transform/src/sroa.rs index c2108795372..3c5ccc0c99a 100644 --- a/compiler/rustc_mir_transform/src/sroa.rs +++ b/compiler/rustc_mir_transform/src/sroa.rs @@ -9,10 +9,11 @@ use rustc_middle::mir::*; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_mir_dataflow::value_analysis::{excluded_locals, iter_fields}; use rustc_target::abi::{FieldIdx, FIRST_VARIANT}; +use tracing::{debug, instrument}; pub struct ScalarReplacementOfAggregates; -impl<'tcx> MirPass<'tcx> for ScalarReplacementOfAggregates { +impl<'tcx> crate::MirPass<'tcx> for ScalarReplacementOfAggregates { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { sess.mir_opt_level() >= 2 } diff --git a/compiler/rustc_mir_transform/src/ssa.rs b/compiler/rustc_mir_transform/src/ssa.rs index 76591f52625..c1254766f0a 100644 --- a/compiler/rustc_mir_transform/src/ssa.rs +++ b/compiler/rustc_mir_transform/src/ssa.rs @@ -14,6 +14,7 @@ use rustc_middle::middle::resolve_bound_vars::Set1; use rustc_middle::mir::visit::*; use rustc_middle::mir::*; use rustc_middle::ty::{ParamEnv, TyCtxt}; +use tracing::{debug, instrument, trace}; pub struct SsaLocals { /// Assignments to each local. This defines whether the local is SSA. diff --git a/compiler/rustc_mir_transform/src/unreachable_enum_branching.rs b/compiler/rustc_mir_transform/src/unreachable_enum_branching.rs index 81baf58a5e0..51a322628ee 100644 --- a/compiler/rustc_mir_transform/src/unreachable_enum_branching.rs +++ b/compiler/rustc_mir_transform/src/unreachable_enum_branching.rs @@ -10,8 +10,7 @@ use rustc_middle::mir::{ use rustc_middle::ty::layout::TyAndLayout; use rustc_middle::ty::{Ty, TyCtxt}; use rustc_target::abi::{Abi, Variants}; - -use crate::MirPass; +use tracing::trace; pub struct UnreachableEnumBranching; @@ -73,7 +72,7 @@ fn variant_discriminants<'tcx>( } } -impl<'tcx> MirPass<'tcx> for UnreachableEnumBranching { +impl<'tcx> crate::MirPass<'tcx> for UnreachableEnumBranching { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { sess.mir_opt_level() > 0 } diff --git a/compiler/rustc_mir_transform/src/unreachable_prop.rs b/compiler/rustc_mir_transform/src/unreachable_prop.rs index a6c3c3b189e..b8da86f1a8d 100644 --- a/compiler/rustc_mir_transform/src/unreachable_prop.rs +++ b/compiler/rustc_mir_transform/src/unreachable_prop.rs @@ -12,7 +12,7 @@ use rustc_target::abi::Size; pub struct UnreachablePropagation; -impl MirPass<'_> for UnreachablePropagation { +impl crate::MirPass<'_> for UnreachablePropagation { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { // Enable only under -Zmir-opt-level=2 as this can make programs less debuggable. sess.mir_opt_level() >= 2 diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index 36908036796..69e2592e82c 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -36,7 +36,7 @@ pub struct Validator { pub mir_phase: MirPhase, } -impl<'tcx> MirPass<'tcx> for Validator { +impl<'tcx> crate::MirPass<'tcx> for Validator { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // FIXME(JakobDegen): These bodies never instantiated in codegend anyway, so it's not // terribly important that they pass the validator. However, I think other passes might @@ -102,25 +102,6 @@ impl<'tcx> MirPass<'tcx> for Validator { } } } - - // Enforce that coroutine-closure layouts are identical. - if let Some(layout) = body.coroutine_layout_raw() - && let Some(by_move_body) = body.coroutine_by_move_body() - && let Some(by_move_layout) = by_move_body.coroutine_layout_raw() - { - // FIXME(async_closures): We could do other validation here? - if layout.variant_fields.len() != by_move_layout.variant_fields.len() { - cfg_checker.fail( - Location::START, - format!( - "Coroutine layout has different number of variant fields from \ - by-move coroutine layout:\n\ - layout: {layout:#?}\n\ - by_move_layout: {by_move_layout:#?}", - ), - ); - } - } } } |