diff options
Diffstat (limited to 'compiler/rustc_mir_transform/src')
47 files changed, 812 insertions, 495 deletions
diff --git a/compiler/rustc_mir_transform/src/add_retag.rs b/compiler/rustc_mir_transform/src/add_retag.rs index 7a8dee09c29..28a5a22dd9d 100644 --- a/compiler/rustc_mir_transform/src/add_retag.rs +++ b/compiler/rustc_mir_transform/src/add_retag.rs @@ -34,7 +34,7 @@ fn is_stable(place: PlaceRef<'_>) -> bool { } /// Determine whether this type may be a reference (or box), and thus needs retagging. -fn may_be_reference(ty: Ty<'tcx>) -> bool { +fn may_be_reference(ty: Ty<'_>) -> bool { match ty.kind() { // Primitive types that are not references ty::Bool @@ -58,11 +58,11 @@ fn may_be_reference(ty: Ty<'tcx>) -> bool { } impl<'tcx> MirPass<'tcx> for AddRetag { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if !tcx.sess.opts.debugging_opts.mir_emit_retag { - return; - } + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.opts.debugging_opts.mir_emit_retag + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // We need an `AllCallEdges` pass before we can do any work. super::add_call_guards::AllCallEdges.run_pass(tcx, body); diff --git a/compiler/rustc_mir_transform/src/check_const_item_mutation.rs b/compiler/rustc_mir_transform/src/check_const_item_mutation.rs index 27fe80a456f..a19a3c8b1d5 100644 --- a/compiler/rustc_mir_transform/src/check_const_item_mutation.rs +++ b/compiler/rustc_mir_transform/src/check_const_item_mutation.rs @@ -6,12 +6,12 @@ use rustc_middle::ty::TyCtxt; use rustc_session::lint::builtin::CONST_ITEM_MUTATION; use rustc_span::def_id::DefId; -use crate::MirPass; +use crate::MirLint; pub struct CheckConstItemMutation; -impl<'tcx> MirPass<'tcx> for CheckConstItemMutation { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { +impl<'tcx> MirLint<'tcx> for CheckConstItemMutation { + fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) { let mut checker = ConstMutationChecker { body, tcx, target_local: None }; checker.visit_body(&body); } @@ -23,7 +23,7 @@ struct ConstMutationChecker<'a, 'tcx> { target_local: Option<Local>, } -impl<'a, 'tcx> ConstMutationChecker<'a, 'tcx> { +impl<'tcx> ConstMutationChecker<'_, 'tcx> { fn is_const_item(&self, local: Local) -> Option<DefId> { if let Some(box LocalInfo::ConstRef { def_id }) = self.body.local_decls[local].local_info { Some(def_id) @@ -95,7 +95,7 @@ impl<'a, 'tcx> ConstMutationChecker<'a, 'tcx> { } } -impl<'a, 'tcx> Visitor<'tcx> for ConstMutationChecker<'a, 'tcx> { +impl<'tcx> Visitor<'tcx> for ConstMutationChecker<'_, 'tcx> { fn visit_statement(&mut self, stmt: &Statement<'tcx>, loc: Location) { if let StatementKind::Assign(box (lhs, _)) = &stmt.kind { // Check for assignment to fields of a constant diff --git a/compiler/rustc_mir_transform/src/check_packed_ref.rs b/compiler/rustc_mir_transform/src/check_packed_ref.rs index 49be34c7a28..23d59c80071 100644 --- a/compiler/rustc_mir_transform/src/check_packed_ref.rs +++ b/compiler/rustc_mir_transform/src/check_packed_ref.rs @@ -7,7 +7,7 @@ use rustc_session::lint::builtin::UNALIGNED_REFERENCES; use rustc_span::symbol::sym; use crate::util; -use crate::MirPass; +use crate::MirLint; pub(crate) fn provide(providers: &mut Providers) { *providers = Providers { unsafe_derive_on_repr_packed, ..*providers }; @@ -15,8 +15,8 @@ pub(crate) fn provide(providers: &mut Providers) { pub struct CheckPackedRef; -impl<'tcx> MirPass<'tcx> for CheckPackedRef { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { +impl<'tcx> MirLint<'tcx> for CheckPackedRef { + fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) { let param_env = tcx.param_env(body.source.def_id()); let source_info = SourceInfo::outermost(body.span); let mut checker = PackedRefChecker { body, tcx, param_env, source_info }; @@ -66,7 +66,7 @@ fn builtin_derive_def_id(tcx: TyCtxt<'_>, def_id: DefId) -> Option<DefId> { } } -impl<'a, 'tcx> Visitor<'tcx> for PackedRefChecker<'a, 'tcx> { +impl<'tcx> Visitor<'tcx> for PackedRefChecker<'_, 'tcx> { fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) { // Make sure we know where in the MIR we are. self.source_info = terminator.source_info; @@ -105,6 +105,11 @@ impl<'a, 'tcx> Visitor<'tcx> for PackedRefChecker<'a, 'tcx> { a misaligned reference is undefined behavior (even if that \ reference is never dereferenced)", ) + .help( + "copy the field contents to a local variable, or replace the \ + reference with a raw pointer and use `read_unaligned`/`write_unaligned` \ + (loads and stores via `*p` must be properly aligned even when using raw pointers)" + ) .emit() }, ); diff --git a/compiler/rustc_mir_transform/src/check_unsafety.rs b/compiler/rustc_mir_transform/src/check_unsafety.rs index 1ff9bd15721..a40c4d1c366 100644 --- a/compiler/rustc_mir_transform/src/check_unsafety.rs +++ b/compiler/rustc_mir_transform/src/check_unsafety.rs @@ -46,7 +46,7 @@ impl<'a, 'tcx> UnsafetyChecker<'a, 'tcx> { } } -impl<'a, 'tcx> Visitor<'tcx> for UnsafetyChecker<'a, 'tcx> { +impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> { fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) { self.source_info = terminator.source_info; match terminator.kind { @@ -117,8 +117,8 @@ impl<'a, 'tcx> Visitor<'tcx> for UnsafetyChecker<'a, 'tcx> { match rvalue { Rvalue::Aggregate(box ref aggregate, _) => match aggregate { &AggregateKind::Array(..) | &AggregateKind::Tuple => {} - &AggregateKind::Adt(ref def, ..) => { - match self.tcx.layout_scalar_valid_range(def.did) { + &AggregateKind::Adt(adt_did, ..) => { + match self.tcx.layout_scalar_valid_range(adt_did) { (Bound::Unbounded, Bound::Unbounded) => {} _ => self.require_unsafe( UnsafetyViolationKind::General, @@ -208,6 +208,7 @@ impl<'a, 'tcx> Visitor<'tcx> for UnsafetyChecker<'a, 'tcx> { MutatingUseContext::Store | MutatingUseContext::Drop | MutatingUseContext::AsmOutput + | MutatingUseContext::LlvmAsmOutput ) ); // If this is just an assignment, determine if the assigned type needs dropping. @@ -243,7 +244,7 @@ impl<'a, 'tcx> Visitor<'tcx> for UnsafetyChecker<'a, 'tcx> { } } -impl<'a, 'tcx> UnsafetyChecker<'a, 'tcx> { +impl<'tcx> UnsafetyChecker<'_, 'tcx> { fn require_unsafe(&mut self, kind: UnsafetyViolationKind, details: UnsafetyViolationDetails) { // Violations can turn out to be `UnsafeFn` during analysis, but they should not start out as such. assert_ne!(kind, UnsafetyViolationKind::UnsafeFn); @@ -396,7 +397,7 @@ struct UnusedUnsafeVisitor<'a> { unsafe_blocks: &'a mut Vec<(hir::HirId, bool)>, } -impl<'a, 'tcx> intravisit::Visitor<'tcx> for UnusedUnsafeVisitor<'a> { +impl<'tcx> intravisit::Visitor<'tcx> for UnusedUnsafeVisitor<'_> { type Map = intravisit::ErasedMap<'tcx>; fn nested_visit_map(&mut self) -> intravisit::NestedVisitorMap<Self::Map> { diff --git a/compiler/rustc_mir_transform/src/const_debuginfo.rs b/compiler/rustc_mir_transform/src/const_debuginfo.rs index b613634560f..839d94167fe 100644 --- a/compiler/rustc_mir_transform/src/const_debuginfo.rs +++ b/compiler/rustc_mir_transform/src/const_debuginfo.rs @@ -15,11 +15,11 @@ use rustc_index::{bit_set::BitSet, vec::IndexVec}; pub struct ConstDebugInfo; impl<'tcx> MirPass<'tcx> for ConstDebugInfo { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if !tcx.sess.opts.debugging_opts.unsound_mir_opts { - return; - } + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.opts.debugging_opts.unsound_mir_opts && sess.mir_opt_level() > 0 + } + fn run_pass(&self, _: TyCtxt<'tcx>, body: &mut Body<'tcx>) { trace!("running ConstDebugInfo on {:?}", body.source); for (local, constant) in find_optimization_oportunities(body) { @@ -89,7 +89,7 @@ fn find_optimization_oportunities<'tcx>(body: &Body<'tcx>) -> Vec<(Local, Consta eligable_locals } -impl<'tcx> Visitor<'tcx> for LocalUseVisitor { +impl Visitor<'_> for LocalUseVisitor { fn visit_local(&mut self, local: &Local, context: PlaceContext, location: Location) { if context.is_mutating_use() { self.local_mutating_uses[*local] = self.local_mutating_uses[*local].saturating_add(1); diff --git a/compiler/rustc_mir_transform/src/const_goto.rs b/compiler/rustc_mir_transform/src/const_goto.rs index d319fdcaa6b..905173b0457 100644 --- a/compiler/rustc_mir_transform/src/const_goto.rs +++ b/compiler/rustc_mir_transform/src/const_goto.rs @@ -27,10 +27,11 @@ use super::simplify::{simplify_cfg, simplify_locals}; pub struct ConstGoto; impl<'tcx> MirPass<'tcx> for ConstGoto { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() >= 4 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if tcx.sess.mir_opt_level() < 4 { - return; - } trace!("Running ConstGoto on {:?}", body.source); let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); let mut opt_finder = @@ -53,7 +54,7 @@ impl<'tcx> MirPass<'tcx> for ConstGoto { } } -impl<'a, 'tcx> Visitor<'tcx> for ConstGotoOptimizationFinder<'a, 'tcx> { +impl<'tcx> Visitor<'tcx> for ConstGotoOptimizationFinder<'_, 'tcx> { fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) { let _: Option<_> = try { let target = terminator.kind.as_goto()?; @@ -82,20 +83,7 @@ impl<'a, 'tcx> Visitor<'tcx> for ConstGotoOptimizationFinder<'a, 'tcx> { // Now find which value in the Switch matches the const value. let const_value = _const.literal.try_eval_bits(self.tcx, self.param_env, switch_ty)?; - let found_value_idx_option = targets - .iter() - .enumerate() - .find(|(_, (value, _))| const_value == *value) - .map(|(idx, _)| idx); - - let target_to_use_in_goto = - if let Some(found_value_idx) = found_value_idx_option { - targets.iter().nth(found_value_idx).unwrap().1 - } else { - // If we did not find the const value in values, it must be the otherwise case - targets.otherwise() - }; - + let target_to_use_in_goto = targets.target_for_value(const_value); self.optimizations.push(OptimizationToApply { bb_with_goto: location.block, target_to_use_in_goto, diff --git a/compiler/rustc_mir_transform/src/const_prop.rs b/compiler/rustc_mir_transform/src/const_prop.rs index 63c637af5c2..7e56e062fc9 100644 --- a/compiler/rustc_mir_transform/src/const_prop.rs +++ b/compiler/rustc_mir_transform/src/const_prop.rs @@ -62,6 +62,13 @@ macro_rules! throw_machine_stop_str { pub struct ConstProp; impl<'tcx> MirPass<'tcx> for ConstProp { + fn is_enabled(&self, _sess: &rustc_session::Session) -> bool { + // FIXME(#70073): Unlike the other passes in "optimizations", this one emits errors, so it + // runs even when MIR optimizations are disabled. We should separate the lint out from the + // transform and move the lint as early in the pipeline as possible. + true + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // will be evaluated by miri and produce its errors there if body.source.promoted.is_some() { @@ -164,7 +171,7 @@ struct ConstPropMachine<'mir, 'tcx> { can_const_prop: IndexVec<Local, ConstPropMode>, } -impl<'mir, 'tcx> ConstPropMachine<'mir, 'tcx> { +impl ConstPropMachine<'_, '_> { fn new( only_propagate_inside_block_locals: BitSet<Local>, can_const_prop: IndexVec<Local, ConstPropMode>, @@ -200,7 +207,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> _args: &[OpTy<'tcx>], _ret: Option<(&PlaceTy<'tcx>, BasicBlock)>, _unwind: StackPopUnwind, - ) -> InterpResult<'tcx, Option<&'mir Body<'tcx>>> { + ) -> InterpResult<'tcx, Option<(&'mir Body<'tcx>, ty::Instance<'tcx>)>> { Ok(None) } @@ -301,14 +308,14 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> } #[inline(always)] - fn stack( + fn stack<'a>( ecx: &'a InterpCx<'mir, 'tcx, Self>, ) -> &'a [Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>] { &ecx.machine.stack } #[inline(always)] - fn stack_mut( + fn stack_mut<'a>( ecx: &'a mut InterpCx<'mir, 'tcx, Self>, ) -> &'a mut Vec<Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>> { &mut ecx.machine.stack @@ -329,7 +336,7 @@ struct ConstPropagator<'mir, 'tcx> { source_info: Option<SourceInfo>, } -impl<'mir, 'tcx> LayoutOfHelpers<'tcx> for ConstPropagator<'mir, 'tcx> { +impl<'tcx> LayoutOfHelpers<'tcx> for ConstPropagator<'_, 'tcx> { type LayoutOfResult = Result<TyAndLayout<'tcx>, LayoutError<'tcx>>; #[inline] @@ -338,21 +345,21 @@ impl<'mir, 'tcx> LayoutOfHelpers<'tcx> for ConstPropagator<'mir, 'tcx> { } } -impl<'mir, 'tcx> HasDataLayout for ConstPropagator<'mir, 'tcx> { +impl HasDataLayout for ConstPropagator<'_, '_> { #[inline] fn data_layout(&self) -> &TargetDataLayout { &self.tcx.data_layout } } -impl<'mir, 'tcx> ty::layout::HasTyCtxt<'tcx> for ConstPropagator<'mir, 'tcx> { +impl<'tcx> ty::layout::HasTyCtxt<'tcx> for ConstPropagator<'_, 'tcx> { #[inline] fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } } -impl<'mir, 'tcx> ty::layout::HasParamEnv<'tcx> for ConstPropagator<'mir, 'tcx> { +impl<'tcx> ty::layout::HasParamEnv<'tcx> for ConstPropagator<'_, 'tcx> { #[inline] fn param_env(&self) -> ty::ParamEnv<'tcx> { self.param_env @@ -745,62 +752,44 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { rvalue: &Rvalue<'tcx>, place: Place<'tcx>, ) -> Option<()> { - self.use_ecx(|this| { - match rvalue { - Rvalue::BinaryOp(op, box (left, right)) - | Rvalue::CheckedBinaryOp(op, box (left, right)) => { - let l = this.ecx.eval_operand(left, None); - let r = this.ecx.eval_operand(right, None); - - let const_arg = match (l, r) { - (Ok(ref x), Err(_)) | (Err(_), Ok(ref x)) => this.ecx.read_immediate(x)?, - (Err(e), Err(_)) => return Err(e), - (Ok(_), Ok(_)) => { - this.ecx.eval_rvalue_into_place(rvalue, place)?; - return Ok(()); - } - }; - - let arg_value = const_arg.to_scalar()?.to_bits(const_arg.layout.size)?; - let dest = this.ecx.eval_place(place)?; - - match op { - BinOp::BitAnd => { - if arg_value == 0 { - this.ecx.write_immediate(*const_arg, &dest)?; - } - } - BinOp::BitOr => { - if arg_value == const_arg.layout.size.truncate(u128::MAX) - || (const_arg.layout.ty.is_bool() && arg_value == 1) - { - this.ecx.write_immediate(*const_arg, &dest)?; - } - } - BinOp::Mul => { - if const_arg.layout.ty.is_integral() && arg_value == 0 { - if let Rvalue::CheckedBinaryOp(_, _) = rvalue { - let val = Immediate::ScalarPair( - const_arg.to_scalar()?.into(), - Scalar::from_bool(false).into(), - ); - this.ecx.write_immediate(val, &dest)?; - } else { - this.ecx.write_immediate(*const_arg, &dest)?; - } - } - } - _ => { - this.ecx.eval_rvalue_into_place(rvalue, place)?; + self.use_ecx(|this| match rvalue { + Rvalue::BinaryOp(op, box (left, right)) + | Rvalue::CheckedBinaryOp(op, box (left, right)) => { + let l = this.ecx.eval_operand(left, None); + let r = this.ecx.eval_operand(right, None); + + let const_arg = match (l, r) { + (Ok(ref x), Err(_)) | (Err(_), Ok(ref x)) => this.ecx.read_immediate(x)?, + (Err(e), Err(_)) => return Err(e), + (Ok(_), Ok(_)) => return this.ecx.eval_rvalue_into_place(rvalue, place), + }; + + let arg_value = const_arg.to_scalar()?.to_bits(const_arg.layout.size)?; + let dest = this.ecx.eval_place(place)?; + + match op { + BinOp::BitAnd if arg_value == 0 => this.ecx.write_immediate(*const_arg, &dest), + BinOp::BitOr + if arg_value == const_arg.layout.size.truncate(u128::MAX) + || (const_arg.layout.ty.is_bool() && arg_value == 1) => + { + this.ecx.write_immediate(*const_arg, &dest) + } + BinOp::Mul if const_arg.layout.ty.is_integral() && arg_value == 0 => { + if let Rvalue::CheckedBinaryOp(_, _) = rvalue { + let val = Immediate::ScalarPair( + const_arg.to_scalar()?.into(), + Scalar::from_bool(false).into(), + ); + this.ecx.write_immediate(val, &dest) + } else { + this.ecx.write_immediate(*const_arg, &dest) } } - } - _ => { - this.ecx.eval_rvalue_into_place(rvalue, place)?; + _ => this.ecx.eval_rvalue_into_place(rvalue, place), } } - - Ok(()) + _ => this.ecx.eval_rvalue_into_place(rvalue, place), }) } @@ -964,7 +953,7 @@ struct CanConstProp { impl CanConstProp { /// Returns true if `local` can be propagated - fn check( + fn check<'tcx>( tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, body: &Body<'tcx>, @@ -1012,7 +1001,7 @@ impl CanConstProp { } } -impl<'tcx> Visitor<'tcx> for CanConstProp { +impl Visitor<'_> for CanConstProp { fn visit_local(&mut self, &local: &Local, context: PlaceContext, _: Location) { use rustc_middle::mir::visit::PlaceContext::*; match context { @@ -1022,6 +1011,7 @@ impl<'tcx> Visitor<'tcx> for CanConstProp { // These are just stores, where the storing is not propagatable, but there may be later // mutations of the same local via `Store` | MutatingUse(MutatingUseContext::Call) + | MutatingUse(MutatingUseContext::AsmOutput) // Actual store that can possibly even propagate a value | MutatingUse(MutatingUseContext::Store) => { if !self.found_assignment.insert(local) { @@ -1052,7 +1042,7 @@ impl<'tcx> Visitor<'tcx> for CanConstProp { // These could be propagated with a smarter analysis or just some careful thinking about // whether they'd be fine right now. - MutatingUse(MutatingUseContext::AsmOutput) + MutatingUse(MutatingUseContext::LlvmAsmOutput) | MutatingUse(MutatingUseContext::Yield) | MutatingUse(MutatingUseContext::Drop) | MutatingUse(MutatingUseContext::Retag) @@ -1071,7 +1061,7 @@ impl<'tcx> Visitor<'tcx> for CanConstProp { } } -impl<'mir, 'tcx> MutVisitor<'tcx> for ConstPropagator<'mir, 'tcx> { +impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } diff --git a/compiler/rustc_mir_transform/src/coverage/debug.rs b/compiler/rustc_mir_transform/src/coverage/debug.rs index 513a85b5913..c61ee6f7e6c 100644 --- a/compiler/rustc_mir_transform/src/coverage/debug.rs +++ b/compiler/rustc_mir_transform/src/coverage/debug.rs @@ -148,7 +148,7 @@ impl DebugOptions { let mut counter_format = ExpressionFormat::default(); if let Ok(env_debug_options) = std::env::var(RUSTC_COVERAGE_DEBUG_OPTIONS) { - for setting_str in env_debug_options.replace(" ", "").replace("-", "_").split(',') { + for setting_str in env_debug_options.replace(' ', "").replace('-', "_").split(',') { let (option, value) = match setting_str.split_once('=') { None => (setting_str, None), Some((k, v)) => (k, Some(v)), @@ -629,7 +629,7 @@ impl UsedExpressions { } /// Generates the MIR pass `CoverageSpan`-specific spanview dump file. -pub(super) fn dump_coverage_spanview( +pub(super) fn dump_coverage_spanview<'tcx>( tcx: TyCtxt<'tcx>, mir_body: &mir::Body<'tcx>, basic_coverage_blocks: &CoverageGraph, @@ -651,7 +651,7 @@ pub(super) fn dump_coverage_spanview( } /// Converts the computed `BasicCoverageBlockData`s into `SpanViewable`s. -fn span_viewables( +fn span_viewables<'tcx>( tcx: TyCtxt<'tcx>, mir_body: &mir::Body<'tcx>, basic_coverage_blocks: &CoverageGraph, @@ -670,7 +670,7 @@ fn span_viewables( } /// Generates the MIR pass coverage-specific graphviz dump file. -pub(super) fn dump_coverage_graphviz( +pub(super) fn dump_coverage_graphviz<'tcx>( tcx: TyCtxt<'tcx>, mir_body: &mir::Body<'tcx>, pass_name: &str, @@ -750,7 +750,7 @@ pub(super) fn dump_coverage_graphviz( .expect("Unexpected error writing BasicCoverageBlock graphviz DOT file"); } -fn bcb_to_string_sections( +fn bcb_to_string_sections<'tcx>( tcx: TyCtxt<'tcx>, mir_body: &mir::Body<'tcx>, debug_counters: &DebugCounters, @@ -817,7 +817,7 @@ fn bcb_to_string_sections( /// Returns a simple string representation of a `TerminatorKind` variant, independent of any /// values it might hold. -pub(super) fn term_type(kind: &TerminatorKind<'tcx>) -> &'static str { +pub(super) fn term_type(kind: &TerminatorKind<'_>) -> &'static str { match kind { TerminatorKind::Goto { .. } => "Goto", TerminatorKind::SwitchInt { .. } => "SwitchInt", diff --git a/compiler/rustc_mir_transform/src/coverage/graph.rs b/compiler/rustc_mir_transform/src/coverage/graph.rs index d78ad6ce97f..a25402a1ff9 100644 --- a/compiler/rustc_mir_transform/src/coverage/graph.rs +++ b/compiler/rustc_mir_transform/src/coverage/graph.rs @@ -27,7 +27,7 @@ pub(super) struct CoverageGraph { } impl CoverageGraph { - pub fn from_mir(mir_body: &mir::Body<'tcx>) -> Self { + pub fn from_mir(mir_body: &mir::Body<'_>) -> Self { let (bcbs, bb_to_bcb) = Self::compute_basic_coverage_blocks(mir_body); // Pre-transform MIR `BasicBlock` successors and predecessors into the BasicCoverageBlock @@ -74,7 +74,7 @@ impl CoverageGraph { } fn compute_basic_coverage_blocks( - mir_body: &mir::Body<'tcx>, + mir_body: &mir::Body<'_>, ) -> ( IndexVec<BasicCoverageBlock, BasicCoverageBlockData>, IndexVec<BasicBlock, Option<BasicCoverageBlock>>, @@ -267,7 +267,7 @@ impl graph::WithSuccessors for CoverageGraph { } } -impl graph::GraphPredecessors<'graph> for CoverageGraph { +impl<'graph> graph::GraphPredecessors<'graph> for CoverageGraph { type Item = BasicCoverageBlock; type Iter = std::iter::Copied<std::slice::Iter<'graph, BasicCoverageBlock>>; } diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs index 6807d02519e..b009e2fd0e4 100644 --- a/compiler/rustc_mir_transform/src/coverage/mod.rs +++ b/compiler/rustc_mir_transform/src/coverage/mod.rs @@ -49,6 +49,10 @@ impl Error { pub struct InstrumentCoverage; impl<'tcx> MirPass<'tcx> for InstrumentCoverage { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.instrument_coverage() + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, mir_body: &mut mir::Body<'tcx>) { let mir_source = mir_body.source; @@ -439,7 +443,7 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { } fn inject_edge_counter_basic_block( - mir_body: &mut mir::Body<'tcx>, + mir_body: &mut mir::Body<'_>, from_bb: BasicBlock, to_bb: BasicBlock, ) -> BasicBlock { @@ -462,7 +466,7 @@ fn inject_edge_counter_basic_block( } fn inject_statement( - mir_body: &mut mir::Body<'tcx>, + mir_body: &mut mir::Body<'_>, counter_kind: CoverageKind, bb: BasicBlock, some_code_region: Option<CodeRegion>, @@ -484,7 +488,7 @@ fn inject_statement( } // Non-code expressions are injected into the coverage map, without generating executable code. -fn inject_intermediate_expression(mir_body: &mut mir::Body<'tcx>, expression: CoverageKind) { +fn inject_intermediate_expression(mir_body: &mut mir::Body<'_>, expression: CoverageKind) { debug_assert!(matches!(expression, CoverageKind::Expression { .. })); debug!(" injecting non-code expression {:?}", expression); let inject_in_bb = mir::START_BLOCK; diff --git a/compiler/rustc_mir_transform/src/coverage/query.rs b/compiler/rustc_mir_transform/src/coverage/query.rs index 760f16eae6b..1721fb5cde0 100644 --- a/compiler/rustc_mir_transform/src/coverage/query.rs +++ b/compiler/rustc_mir_transform/src/coverage/query.rs @@ -137,7 +137,7 @@ fn coverageinfo<'tcx>(tcx: TyCtxt<'tcx>, instance_def: ty::InstanceDef<'tcx>) -> coverage_visitor.info } -fn covered_file_name<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> Option<Symbol> { +fn covered_file_name(tcx: TyCtxt<'_>, def_id: DefId) -> Option<Symbol> { if tcx.is_mir_available(def_id) { let body = mir_body(tcx, def_id); for bb_data in body.basic_blocks().iter() { diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs index d13fa0729cd..b5356a817f7 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans.rs @@ -21,7 +21,7 @@ pub(super) enum CoverageStatement { } impl CoverageStatement { - pub fn format(&self, tcx: TyCtxt<'tcx>, mir_body: &'a mir::Body<'tcx>) -> String { + pub fn format<'tcx>(&self, tcx: TyCtxt<'tcx>, mir_body: &mir::Body<'tcx>) -> String { match *self { Self::Statement(bb, span, stmt_index) => { let stmt = &mir_body[bb].statements[stmt_index]; @@ -86,7 +86,7 @@ impl CoverageSpan { } pub fn for_statement( - statement: &Statement<'tcx>, + statement: &Statement<'_>, span: Span, expn_span: Span, bcb: BasicCoverageBlock, @@ -151,18 +151,18 @@ impl CoverageSpan { self.bcb == other.bcb } - pub fn format(&self, tcx: TyCtxt<'tcx>, mir_body: &'a mir::Body<'tcx>) -> String { + pub fn format<'tcx>(&self, tcx: TyCtxt<'tcx>, mir_body: &mir::Body<'tcx>) -> String { format!( "{}\n {}", source_range_no_file(tcx, &self.span), - self.format_coverage_statements(tcx, mir_body).replace("\n", "\n "), + self.format_coverage_statements(tcx, mir_body).replace('\n', "\n "), ) } - pub fn format_coverage_statements( + pub fn format_coverage_statements<'tcx>( &self, tcx: TyCtxt<'tcx>, - mir_body: &'a mir::Body<'tcx>, + mir_body: &mir::Body<'tcx>, ) -> String { let mut sorted_coverage_statements = self.coverage_statements.clone(); sorted_coverage_statements.sort_unstable_by_key(|covstmt| match *covstmt { @@ -329,9 +329,7 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { fn mir_to_initial_sorted_coverage_spans(&self) -> Vec<CoverageSpan> { let mut initial_spans = Vec::<CoverageSpan>::with_capacity(self.mir_body.num_nodes() * 2); for (bcb, bcb_data) in self.basic_coverage_blocks.iter_enumerated() { - for coverage_span in self.bcb_to_initial_coverage_spans(bcb, bcb_data) { - initial_spans.push(coverage_span); - } + initial_spans.extend(self.bcb_to_initial_coverage_spans(bcb, bcb_data)); } if initial_spans.is_empty() { @@ -803,7 +801,7 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { /// If the MIR `Statement` has a span contributive to computing coverage spans, /// return it; otherwise return `None`. -pub(super) fn filtered_statement_span(statement: &'a Statement<'tcx>) -> Option<Span> { +pub(super) fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span> { match statement.kind { // These statements have spans that are often outside the scope of the executed source code // for their parent `BasicBlock`. @@ -847,7 +845,7 @@ pub(super) fn filtered_statement_span(statement: &'a Statement<'tcx>) -> Option< /// If the MIR `Terminator` has a span contributive to computing coverage spans, /// return it; otherwise return `None`. -pub(super) fn filtered_terminator_span(terminator: &'a Terminator<'tcx>) -> Option<Span> { +pub(super) fn filtered_terminator_span(terminator: &Terminator<'_>) -> Option<Span> { match terminator.kind { // These terminators have spans that don't positively contribute to computing a reasonable // span of actually executed source code. (For example, SwitchInt terminators extracted from diff --git a/compiler/rustc_mir_transform/src/coverage/tests.rs b/compiler/rustc_mir_transform/src/coverage/tests.rs index 14dd0a8b924..b9c79d4cf2d 100644 --- a/compiler/rustc_mir_transform/src/coverage/tests.rs +++ b/compiler/rustc_mir_transform/src/coverage/tests.rs @@ -180,7 +180,7 @@ impl<'tcx> MockBlocks<'tcx> { } } -fn debug_basic_blocks(mir_body: &Body<'tcx>) -> String { +fn debug_basic_blocks<'tcx>(mir_body: &Body<'tcx>) -> String { format!( "{:?}", mir_body @@ -273,7 +273,7 @@ fn print_coverage_graphviz( } /// Create a mock `Body` with a simple flow. -fn goto_switchint() -> Body<'a> { +fn goto_switchint<'a>() -> Body<'a> { let mut blocks = MockBlocks::new(); let start = blocks.call(None); let goto = blocks.goto(Some(start)); @@ -363,7 +363,7 @@ fn test_covgraph_goto_switchint() { } /// Create a mock `Body` with a loop. -fn switchint_then_loop_else_return() -> Body<'a> { +fn switchint_then_loop_else_return<'a>() -> Body<'a> { let mut blocks = MockBlocks::new(); let start = blocks.call(None); let switchint = blocks.switchint(Some(start)); @@ -449,7 +449,7 @@ fn test_covgraph_switchint_then_loop_else_return() { } /// Create a mock `Body` with nested loops. -fn switchint_loop_then_inner_loop_else_break() -> Body<'a> { +fn switchint_loop_then_inner_loop_else_break<'a>() -> Body<'a> { let mut blocks = MockBlocks::new(); let start = blocks.call(None); let switchint = blocks.switchint(Some(start)); diff --git a/compiler/rustc_mir_transform/src/deduplicate_blocks.rs b/compiler/rustc_mir_transform/src/deduplicate_blocks.rs index 8d2413433a9..d1977ed49fe 100644 --- a/compiler/rustc_mir_transform/src/deduplicate_blocks.rs +++ b/compiler/rustc_mir_transform/src/deduplicate_blocks.rs @@ -15,10 +15,11 @@ use super::simplify::simplify_cfg; pub struct DeduplicateBlocks; impl<'tcx> MirPass<'tcx> for DeduplicateBlocks { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() >= 4 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if tcx.sess.mir_opt_level() < 4 { - return; - } debug!("Running DeduplicateBlocks on `{:?}`", body.source); let duplicates = find_duplicates(body); let has_opts_to_apply = !duplicates.is_empty(); @@ -53,7 +54,7 @@ impl<'tcx> MutVisitor<'tcx> for OptApplier<'tcx> { } } -fn find_duplicates<'a, 'tcx>(body: &'a Body<'tcx>) -> FxHashMap<BasicBlock, BasicBlock> { +fn find_duplicates(body: &Body<'_>) -> FxHashMap<BasicBlock, BasicBlock> { let mut duplicates = FxHashMap::default(); let bbs_to_go_through = @@ -101,7 +102,7 @@ struct BasicBlockHashable<'tcx, 'a> { basic_block_data: &'a BasicBlockData<'tcx>, } -impl<'tcx, 'a> Hash for BasicBlockHashable<'tcx, 'a> { +impl Hash for BasicBlockHashable<'_, '_> { fn hash<H: Hasher>(&self, state: &mut H) { hash_statements(state, self.basic_block_data.statements.iter()); // Note that since we only hash the kind, we lose span information if we deduplicate the blocks @@ -109,9 +110,9 @@ impl<'tcx, 'a> Hash for BasicBlockHashable<'tcx, 'a> { } } -impl<'tcx, 'a> Eq for BasicBlockHashable<'tcx, 'a> {} +impl Eq for BasicBlockHashable<'_, '_> {} -impl<'tcx, 'a> PartialEq for BasicBlockHashable<'tcx, 'a> { +impl PartialEq for BasicBlockHashable<'_, '_> { fn eq(&self, other: &Self) -> bool { self.basic_block_data.statements.len() == other.basic_block_data.statements.len() && &self.basic_block_data.terminator().kind == &other.basic_block_data.terminator().kind @@ -131,7 +132,7 @@ fn hash_statements<'a, 'tcx, H: Hasher>( } } -fn statement_hash<'tcx, H: Hasher>(hasher: &mut H, stmt: &StatementKind<'tcx>) { +fn statement_hash<H: Hasher>(hasher: &mut H, stmt: &StatementKind<'_>) { match stmt { StatementKind::Assign(box (place, rvalue)) => { place.hash(hasher); @@ -141,14 +142,14 @@ fn statement_hash<'tcx, H: Hasher>(hasher: &mut H, stmt: &StatementKind<'tcx>) { }; } -fn rvalue_hash<H: Hasher>(hasher: &mut H, rvalue: &Rvalue<'tcx>) { +fn rvalue_hash<H: Hasher>(hasher: &mut H, rvalue: &Rvalue<'_>) { match rvalue { Rvalue::Use(op) => operand_hash(hasher, op), x => x.hash(hasher), }; } -fn operand_hash<H: Hasher>(hasher: &mut H, operand: &Operand<'tcx>) { +fn operand_hash<H: Hasher>(hasher: &mut H, operand: &Operand<'_>) { match operand { Operand::Constant(box Constant { user_ty: _, literal, span: _ }) => literal.hash(hasher), x => x.hash(hasher), @@ -167,7 +168,7 @@ fn statement_eq<'tcx>(lhs: &StatementKind<'tcx>, rhs: &StatementKind<'tcx>) -> b res } -fn rvalue_eq(lhs: &Rvalue<'tcx>, rhs: &Rvalue<'tcx>) -> bool { +fn rvalue_eq<'tcx>(lhs: &Rvalue<'tcx>, rhs: &Rvalue<'tcx>) -> bool { let res = match (lhs, rhs) { (Rvalue::Use(op1), Rvalue::Use(op2)) => operand_eq(op1, op2), (x, y) => x == y, @@ -176,7 +177,7 @@ fn rvalue_eq(lhs: &Rvalue<'tcx>, rhs: &Rvalue<'tcx>) -> bool { res } -fn operand_eq(lhs: &Operand<'tcx>, rhs: &Operand<'tcx>) -> bool { +fn operand_eq<'tcx>(lhs: &Operand<'tcx>, rhs: &Operand<'tcx>) -> bool { let res = match (lhs, rhs) { ( Operand::Constant(box Constant { user_ty: _, literal, span: _ }), diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index c45946a9e2a..2b382468be0 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -124,18 +124,15 @@ const MAX_BLOCKS: usize = 250; pub struct DestinationPropagation; impl<'tcx> MirPass<'tcx> for DestinationPropagation { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - // FIXME(#79191, #82678) - if !tcx.sess.opts.debugging_opts.unsound_mir_opts { - return; - } - + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + // FIXME(#79191, #82678): This is unsound. + // // Only run at mir-opt-level=3 or higher for now (we don't fix up debuginfo and remove // storage statements at the moment). - if tcx.sess.mir_opt_level() < 3 { - return; - } + sess.opts.debugging_opts.unsound_mir_opts && sess.mir_opt_level() >= 3 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let def_id = body.source.def_id(); let candidates = find_candidates(tcx, body); @@ -244,7 +241,7 @@ struct Replacements<'tcx> { kill: BitSet<Local>, } -impl Replacements<'tcx> { +impl<'tcx> Replacements<'tcx> { fn new(locals: usize) -> Self { Self { map: IndexVec::from_elem_n(None, locals), kill: BitSet::new_empty(locals) } } @@ -301,7 +298,7 @@ struct Replacer<'tcx> { } impl<'tcx> MutVisitor<'tcx> for Replacer<'tcx> { - fn tcx<'a>(&'a self) -> TyCtxt<'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } @@ -375,7 +372,7 @@ struct Conflicts<'a> { unified_locals: InPlaceUnificationTable<UnifyLocal>, } -impl Conflicts<'a> { +impl<'a> Conflicts<'a> { fn build<'tcx>( tcx: TyCtxt<'tcx>, body: &'_ Body<'tcx>, @@ -624,6 +621,7 @@ impl Conflicts<'a> { options: _, line_spans: _, destination: _, + cleanup: _, } => { // The intended semantics here aren't documented, we just assume that nothing that // could be written to by the assembly may overlap with any other operands. @@ -822,10 +820,7 @@ struct CandidateAssignment<'tcx> { /// comment) and also throw out assignments that involve a local that has its address taken or is /// otherwise ineligible (eg. locals used as array indices are ignored because we cannot propagate /// arbitrary places into array indices). -fn find_candidates<'a, 'tcx>( - tcx: TyCtxt<'tcx>, - body: &'a Body<'tcx>, -) -> Vec<CandidateAssignment<'tcx>> { +fn find_candidates<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> Vec<CandidateAssignment<'tcx>> { let mut visitor = FindAssignments { tcx, body, @@ -845,7 +840,7 @@ struct FindAssignments<'a, 'tcx> { locals_used_as_array_index: BitSet<Local>, } -impl<'a, 'tcx> Visitor<'tcx> for FindAssignments<'a, 'tcx> { +impl<'tcx> Visitor<'tcx> for FindAssignments<'_, 'tcx> { fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { if let StatementKind::Assign(box ( dest, diff --git a/compiler/rustc_mir_transform/src/early_otherwise_branch.rs b/compiler/rustc_mir_transform/src/early_otherwise_branch.rs index f191911a6c7..ac88060f0d3 100644 --- a/compiler/rustc_mir_transform/src/early_otherwise_branch.rs +++ b/compiler/rustc_mir_transform/src/early_otherwise_branch.rs @@ -25,16 +25,14 @@ use super::simplify::simplify_cfg; pub struct EarlyOtherwiseBranch; impl<'tcx> MirPass<'tcx> for EarlyOtherwiseBranch { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { // FIXME(#78496) - if !tcx.sess.opts.debugging_opts.unsound_mir_opts { - return; - } + sess.opts.debugging_opts.unsound_mir_opts && sess.mir_opt_level() >= 3 + } - if tcx.sess.mir_opt_level() < 3 { - return; - } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { trace!("running EarlyOtherwiseBranch on {:?}", body.source); + // we are only interested in this bb if the terminator is a switchInt let bbs_with_switch = body.basic_blocks().iter_enumerated().filter(|(_, bb)| is_switch(bb.terminator())); @@ -169,7 +167,7 @@ impl<'tcx> MirPass<'tcx> for EarlyOtherwiseBranch { } } -fn is_switch<'tcx>(terminator: &Terminator<'tcx>) -> bool { +fn is_switch(terminator: &Terminator<'_>) -> bool { matches!(terminator.kind, TerminatorKind::SwitchInt { .. }) } @@ -210,7 +208,7 @@ struct OptimizationInfo<'tcx> { second_switch_info: SwitchDiscriminantInfo<'tcx>, } -impl<'a, 'tcx> Helper<'a, 'tcx> { +impl<'tcx> Helper<'_, 'tcx> { pub fn go( &self, bb: &BasicBlockData<'tcx>, diff --git a/compiler/rustc_mir_transform/src/elaborate_drops.rs b/compiler/rustc_mir_transform/src/elaborate_drops.rs index b9a48197a35..d346dfb1772 100644 --- a/compiler/rustc_mir_transform/src/elaborate_drops.rs +++ b/compiler/rustc_mir_transform/src/elaborate_drops.rs @@ -19,6 +19,10 @@ use std::fmt; pub struct ElaborateDrops; impl<'tcx> MirPass<'tcx> for ElaborateDrops { + fn phase_change(&self) -> Option<MirPhase> { + Some(MirPhase::DropLowering) + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { debug!("elaborate_drops({:?} @ {:?})", body.source, body.span); @@ -145,13 +149,13 @@ struct Elaborator<'a, 'b, 'tcx> { ctxt: &'a mut ElaborateDropsCtxt<'b, 'tcx>, } -impl<'a, 'b, 'tcx> fmt::Debug for Elaborator<'a, 'b, 'tcx> { +impl fmt::Debug for Elaborator<'_, '_, '_> { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { Ok(()) } } -impl<'a, 'b, 'tcx> DropElaborator<'a, 'tcx> for Elaborator<'a, 'b, 'tcx> { +impl<'a, 'tcx> DropElaborator<'a, 'tcx> for Elaborator<'a, '_, 'tcx> { type Path = MovePathIndex; fn patch(&mut self) -> &mut MirPatch<'tcx> { diff --git a/compiler/rustc_mir_transform/src/function_item_references.rs b/compiler/rustc_mir_transform/src/function_item_references.rs index 996c158c062..05834b443d0 100644 --- a/compiler/rustc_mir_transform/src/function_item_references.rs +++ b/compiler/rustc_mir_transform/src/function_item_references.rs @@ -11,12 +11,12 @@ use rustc_session::lint::builtin::FUNCTION_ITEM_REFERENCES; use rustc_span::{symbol::sym, Span}; use rustc_target::spec::abi::Abi; -use crate::MirPass; +use crate::MirLint; pub struct FunctionItemReferences; -impl<'tcx> MirPass<'tcx> for FunctionItemReferences { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { +impl<'tcx> MirLint<'tcx> for FunctionItemReferences { + fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) { let mut checker = FunctionItemRefChecker { tcx, body }; checker.visit_body(&body); } @@ -27,7 +27,7 @@ struct FunctionItemRefChecker<'a, 'tcx> { body: &'a Body<'tcx>, } -impl<'a, 'tcx> Visitor<'tcx> for FunctionItemRefChecker<'a, 'tcx> { +impl<'tcx> Visitor<'tcx> for FunctionItemRefChecker<'_, 'tcx> { /// Emits a lint for function reference arguments bound by `fmt::Pointer` or passed to /// `transmute`. This only handles arguments in calls outside macro expansions to avoid double /// counting function references formatted as pointers by macros. @@ -92,7 +92,7 @@ impl<'a, 'tcx> Visitor<'tcx> for FunctionItemRefChecker<'a, 'tcx> { } } -impl<'a, 'tcx> FunctionItemRefChecker<'a, 'tcx> { +impl<'tcx> FunctionItemRefChecker<'_, 'tcx> { /// Emits a lint for function reference arguments bound by `fmt::Pointer` in calls to the /// function defined by `def_id` with the substitutions `substs_ref`. fn check_bound_args( diff --git a/compiler/rustc_mir_transform/src/generator.rs b/compiler/rustc_mir_transform/src/generator.rs index bc72e9d94a9..bc9a104e849 100644 --- a/compiler/rustc_mir_transform/src/generator.rs +++ b/compiler/rustc_mir_transform/src/generator.rs @@ -233,7 +233,7 @@ struct TransformVisitor<'tcx> { new_ret_local: Local, } -impl TransformVisitor<'tcx> { +impl<'tcx> TransformVisitor<'tcx> { // Make a GeneratorState variant assignment. `core::ops::GeneratorState` only has single // element tuple variants, so we can just write to the downcasted first field and then set the // discriminant to the appropriate variant. @@ -243,7 +243,7 @@ impl TransformVisitor<'tcx> { val: Operand<'tcx>, source_info: SourceInfo, ) -> impl Iterator<Item = Statement<'tcx>> { - let kind = AggregateKind::Adt(self.state_adt_ref, idx, self.state_substs, None, None); + let kind = AggregateKind::Adt(self.state_adt_ref.did, idx, self.state_substs, None, None); assert_eq!(self.state_adt_ref.variants[idx].fields.len(), 1); let ty = self .tcx @@ -295,7 +295,7 @@ impl TransformVisitor<'tcx> { } } -impl MutVisitor<'tcx> for TransformVisitor<'tcx> { +impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } @@ -446,7 +446,7 @@ struct LivenessInfo { storage_liveness: IndexVec<BasicBlock, Option<BitSet<Local>>>, } -fn locals_live_across_suspend_points( +fn locals_live_across_suspend_points<'tcx>( tcx: TyCtxt<'tcx>, body: &Body<'tcx>, always_live_locals: &storage::AlwaysLiveLocals, @@ -613,7 +613,7 @@ impl ops::Deref for GeneratorSavedLocals { /// time. Generates a bitset for every local of all the other locals that may be /// StorageLive simultaneously with that local. This is used in the layout /// computation; see `GeneratorLayout` for more. -fn compute_storage_conflicts( +fn compute_storage_conflicts<'mir, 'tcx>( body: &'mir Body<'tcx>, saved_locals: &GeneratorSavedLocals, always_live_locals: storage::AlwaysLiveLocals, @@ -672,7 +672,9 @@ struct StorageConflictVisitor<'mir, 'tcx, 's> { local_conflicts: BitMatrix<Local, Local>, } -impl rustc_mir_dataflow::ResultsVisitor<'mir, 'tcx> for StorageConflictVisitor<'mir, 'tcx, '_> { +impl<'mir, 'tcx> rustc_mir_dataflow::ResultsVisitor<'mir, 'tcx> + for StorageConflictVisitor<'mir, 'tcx, '_> +{ type FlowState = BitSet<Local>; fn visit_statement_before_primary_effect( @@ -694,7 +696,7 @@ impl rustc_mir_dataflow::ResultsVisitor<'mir, 'tcx> for StorageConflictVisitor<' } } -impl<'body, 'tcx, 's> StorageConflictVisitor<'body, 'tcx, 's> { +impl StorageConflictVisitor<'_, '_, '_> { fn apply_state(&mut self, flow_state: &BitSet<Local>, loc: Location) { // Ignore unreachable blocks. if self.body.basic_blocks()[loc.block].terminator().kind == TerminatorKind::Unreachable { @@ -1232,6 +1234,10 @@ fn create_cases<'tcx>( } impl<'tcx> MirPass<'tcx> for StateTransform { + fn phase_change(&self) -> Option<MirPhase> { + Some(MirPhase::GeneratorLowering) + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let yield_ty = if let Some(yield_ty) = body.yield_ty() { yield_ty @@ -1394,7 +1400,7 @@ impl EnsureGeneratorFieldAssignmentsNeverAlias<'_> { self.saved_locals.get(place.local) } - fn check_assigned_place(&mut self, place: Place<'tcx>, f: impl FnOnce(&mut Self)) { + fn check_assigned_place(&mut self, place: Place<'_>, f: impl FnOnce(&mut Self)) { if let Some(assigned_local) = self.saved_local_for_direct_place(place) { assert!(self.assigned_local.is_none(), "`check_assigned_place` must not recurse"); @@ -1405,7 +1411,7 @@ impl EnsureGeneratorFieldAssignmentsNeverAlias<'_> { } } -impl Visitor<'tcx> for EnsureGeneratorFieldAssignmentsNeverAlias<'_> { +impl<'tcx> Visitor<'tcx> for EnsureGeneratorFieldAssignmentsNeverAlias<'_> { fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, location: Location) { let lhs = match self.assigned_local { Some(l) => l, diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 84a1e3fb600..e1f30fef44f 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -37,21 +37,16 @@ struct CallSite<'tcx> { source_info: SourceInfo, } -/// Returns true if MIR inlining is enabled in the current compilation session. -crate fn is_enabled(tcx: TyCtxt<'_>) -> bool { - if let Some(enabled) = tcx.sess.opts.debugging_opts.inline_mir { - return enabled; - } - - tcx.sess.mir_opt_level() >= 3 -} - impl<'tcx> MirPass<'tcx> for Inline { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if !is_enabled(tcx) { - return; + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + if let Some(enabled) = sess.opts.debugging_opts.inline_mir { + return enabled; } + sess.opts.mir_opt_level() >= 3 + } + + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let span = trace_span!("inline", body = %tcx.def_path_str(body.source.def_id())); let _guard = span.enter(); if inline(tcx, body) { @@ -62,7 +57,7 @@ impl<'tcx> MirPass<'tcx> for Inline { } } -fn inline(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { +fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { let def_id = body.source.def_id(); let hir_id = tcx.hir().local_def_id_to_hir_id(def_id.expect_local()); @@ -73,6 +68,12 @@ fn inline(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { if body.source.promoted.is_some() { return false; } + // Avoid inlining into generators, since their `optimized_mir` is used for layout computation, + // which can create a cycle, even when no attempt is made to inline the function in the other + // direction. + if body.generator.is_some() { + return false; + } let mut this = Inliner { tcx, @@ -100,7 +101,7 @@ struct Inliner<'tcx> { changed: bool, } -impl Inliner<'tcx> { +impl<'tcx> Inliner<'tcx> { fn process_blocks(&mut self, caller_body: &mut Body<'tcx>, blocks: Range<BasicBlock>) { for bb in blocks { let bb_data = &caller_body[bb]; @@ -207,19 +208,12 @@ impl Inliner<'tcx> { if let Some(callee_def_id) = callee.def_id().as_local() { let callee_hir_id = self.tcx.hir().local_def_id_to_hir_id(callee_def_id); - // Avoid inlining into generators, - // since their `optimized_mir` is used for layout computation, which can - // create a cycle, even when no attempt is made to inline the function - // in the other direction. - if caller_body.generator.is_some() { - return Err("local generator (query cycle avoidance)"); - } - // Avoid a cycle here by only using `instance_mir` only if we have // a lower `HirId` than the callee. This ensures that the callee will // not inline us. This trick only works without incremental compilation. // So don't do it if that is enabled. - if !self.tcx.dep_graph.is_fully_enabled() && self.hir_id < callee_hir_id { + if !self.tcx.dep_graph.is_fully_enabled() && self.hir_id.index() < callee_hir_id.index() + { return Ok(()); } @@ -441,6 +435,13 @@ impl Inliner<'tcx> { } } TerminatorKind::Resume => cost += RESUME_PENALTY, + TerminatorKind::InlineAsm { cleanup, .. } => { + cost += INSTR_COST; + + if cleanup.is_some() { + cost += LANDINGPAD_PENALTY; + } + } _ => cost += INSTR_COST, } @@ -784,7 +785,7 @@ struct Integrator<'a, 'tcx> { always_live_locals: BitSet<Local>, } -impl<'a, 'tcx> Integrator<'a, 'tcx> { +impl Integrator<'_, '_> { fn map_local(&self, local: Local) -> Local { let new = if local == RETURN_PLACE { self.destination.local @@ -813,7 +814,7 @@ impl<'a, 'tcx> Integrator<'a, 'tcx> { } } -impl<'a, 'tcx> MutVisitor<'tcx> for Integrator<'a, 'tcx> { +impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } @@ -954,9 +955,13 @@ impl<'a, 'tcx> MutVisitor<'tcx> for Integrator<'a, 'tcx> { { bug!("False unwinds should have been removed before inlining") } - TerminatorKind::InlineAsm { ref mut destination, .. } => { + TerminatorKind::InlineAsm { ref mut destination, ref mut cleanup, .. } => { if let Some(ref mut tgt) = *destination { *tgt = self.map_block(*tgt); + } else if !self.in_cleanup_block { + // Unless this inline asm is in a cleanup block, add an unwind edge to + // the original call's cleanup block + *cleanup = self.cleanup_block; } } } diff --git a/compiler/rustc_mir_transform/src/inline/cycle.rs b/compiler/rustc_mir_transform/src/inline/cycle.rs index 385394ba67d..747e760a18b 100644 --- a/compiler/rustc_mir_transform/src/inline/cycle.rs +++ b/compiler/rustc_mir_transform/src/inline/cycle.rs @@ -10,7 +10,7 @@ use rustc_session::Limit; // FIXME: check whether it is cheaper to precompute the entire call graph instead of invoking // this query riddiculously often. #[instrument(level = "debug", skip(tcx, root, target))] -crate fn mir_callgraph_reachable( +crate fn mir_callgraph_reachable<'tcx>( tcx: TyCtxt<'tcx>, (root, target): (ty::Instance<'tcx>, LocalDefId), ) -> bool { @@ -33,7 +33,7 @@ crate fn mir_callgraph_reachable( level = "debug", skip(tcx, param_env, target, stack, seen, recursion_limiter, caller, recursion_limit) )] - fn process( + fn process<'tcx>( tcx: TyCtxt<'tcx>, param_env: ty::ParamEnv<'tcx>, caller: ty::Instance<'tcx>, diff --git a/compiler/rustc_mir_transform/src/instcombine.rs b/compiler/rustc_mir_transform/src/instcombine.rs index e15a69c95ae..792ac68671e 100644 --- a/compiler/rustc_mir_transform/src/instcombine.rs +++ b/compiler/rustc_mir_transform/src/instcombine.rs @@ -11,6 +11,10 @@ use rustc_middle::ty::{self, TyCtxt}; pub struct InstCombine; impl<'tcx> MirPass<'tcx> for InstCombine { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() > 0 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let (basic_blocks, local_decls) = body.basic_blocks_and_local_decls_mut(); let ctx = InstCombineContext { tcx, local_decls }; @@ -34,7 +38,7 @@ struct InstCombineContext<'tcx, 'a> { local_decls: &'a LocalDecls<'tcx>, } -impl<'tcx, 'a> InstCombineContext<'tcx, 'a> { +impl<'tcx> InstCombineContext<'tcx, '_> { fn should_combine(&self, source_info: &SourceInfo, rvalue: &Rvalue<'tcx>) -> bool { self.tcx.consider_optimizing(|| { format!("InstCombine - Rvalue: {:?} SourceInfo: {:?}", rvalue, source_info) diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index b0bea7312a7..638baa0b8d3 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -1,8 +1,6 @@ #![feature(box_patterns)] #![feature(box_syntax)] #![feature(crate_visibility_modifier)] -#![feature(in_band_lifetimes)] -#![feature(iter_zip)] #![feature(let_else)] #![feature(map_try_insert)] #![feature(min_specialization)] @@ -27,11 +25,16 @@ use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor}; use rustc_index::vec::IndexVec; use rustc_middle::mir::visit::Visitor as _; -use rustc_middle::mir::{dump_mir, traversal, Body, ConstQualifs, MirPass, MirPhase, Promoted}; +use rustc_middle::mir::{traversal, Body, ConstQualifs, MirPass, MirPhase, Promoted}; use rustc_middle::ty::query::Providers; use rustc_middle::ty::{self, TyCtxt, TypeFoldable}; use rustc_span::{Span, Symbol}; +#[macro_use] +mod pass_manager; + +use pass_manager::{self as pm, Lint, MirLint, WithMinOptLevel}; + mod abort_unwinding_calls; mod add_call_guards; mod add_moves_for_packed_drops; @@ -56,12 +59,15 @@ mod inline; mod instcombine; mod lower_intrinsics; mod lower_slice_len; +mod marker; mod match_branches; mod multiple_return_terminators; mod normalize_array_len; mod nrvo; +mod remove_false_edges; mod remove_noop_landing_pads; mod remove_storage_markers; +mod remove_uninit_drops; mod remove_unneeded_drops; mod remove_zsts; mod required_consts; @@ -75,7 +81,7 @@ mod simplify_try; mod uninhabited_enum_branching; mod unreachable_prop; -use rustc_const_eval::transform::check_consts; +use rustc_const_eval::transform::check_consts::{self, ConstCx}; use rustc_const_eval::transform::promote_consts; use rustc_const_eval::transform::validate; use rustc_mir_dataflow::rustc_peek; @@ -142,7 +148,7 @@ fn mir_keys(tcx: TyCtxt<'_>, (): ()) -> FxHashSet<LocalDefId> { tcx: TyCtxt<'tcx>, set: &'a mut FxHashSet<LocalDefId>, } - impl<'a, 'tcx> Visitor<'tcx> for GatherCtors<'a, 'tcx> { + impl<'tcx> Visitor<'tcx> for GatherCtors<'_, 'tcx> { fn visit_variant_data( &mut self, v: &'tcx hir::VariantData<'tcx>, @@ -166,66 +172,6 @@ fn mir_keys(tcx: TyCtxt<'_>, (): ()) -> FxHashSet<LocalDefId> { set } -fn run_passes( - tcx: TyCtxt<'tcx>, - body: &mut Body<'tcx>, - mir_phase: MirPhase, - passes: &[&[&dyn MirPass<'tcx>]], -) { - let phase_index = mir_phase.phase_index(); - let validate = tcx.sess.opts.debugging_opts.validate_mir; - - if body.phase >= mir_phase { - return; - } - - if validate { - validate::Validator { when: format!("input to phase {:?}", mir_phase), mir_phase } - .run_pass(tcx, body); - } - - let mut index = 0; - let mut run_pass = |pass: &dyn MirPass<'tcx>| { - let run_hooks = |body: &_, index, is_after| { - let disambiguator = if is_after { "after" } else { "before" }; - dump_mir( - tcx, - Some(&format_args!("{:03}-{:03}", phase_index, index)), - &pass.name(), - &disambiguator, - body, - |_, _| Ok(()), - ); - }; - run_hooks(body, index, false); - pass.run_pass(tcx, body); - run_hooks(body, index, true); - - if validate { - validate::Validator { - when: format!("after {} in phase {:?}", pass.name(), mir_phase), - mir_phase, - } - .run_pass(tcx, body); - } - - index += 1; - }; - - for pass_group in passes { - for pass in *pass_group { - run_pass(*pass); - } - } - - body.phase = mir_phase; - - if mir_phase == MirPhase::Optimization { - validate::Validator { when: format!("end of phase {:?}", mir_phase), mir_phase } - .run_pass(tcx, body); - } -} - fn mir_const_qualif(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> ConstQualifs { let const_kind = tcx.hir().body_const_context(def.did); @@ -277,25 +223,25 @@ fn mir_const<'tcx>( rustc_middle::mir::dump_mir(tcx, None, "mir_map", &0, &body, |_, _| Ok(())); - run_passes( + pm::run_passes( tcx, &mut body, - MirPhase::Const, - &[&[ + &[ // MIR-level lints. - &check_packed_ref::CheckPackedRef, - &check_const_item_mutation::CheckConstItemMutation, - &function_item_references::FunctionItemReferences, + &Lint(check_packed_ref::CheckPackedRef), + &Lint(check_const_item_mutation::CheckConstItemMutation), + &Lint(function_item_references::FunctionItemReferences), // What we need to do constant evaluation. &simplify::SimplifyCfg::new("initial"), - &rustc_peek::SanityCheck, - ]], + &rustc_peek::SanityCheck, // Just a lint + &marker::PhaseChange(MirPhase::Const), + ], ); tcx.alloc_steal_mir(body) } /// Compute the main MIR body and the list of MIR bodies of the promoteds. -fn mir_promoted( +fn mir_promoted<'tcx>( tcx: TyCtxt<'tcx>, def: ty::WithOptConstParam<LocalDefId>, ) -> (&'tcx Steal<Body<'tcx>>, &'tcx Steal<IndexVec<Promoted, Body<'tcx>>>) { @@ -316,17 +262,17 @@ fn mir_promoted( } body.required_consts = required_consts; + // What we need to run borrowck etc. let promote_pass = promote_consts::PromoteTemps::default(); - let promote: &[&dyn MirPass<'tcx>] = &[ - // What we need to run borrowck etc. - &promote_pass, - &simplify::SimplifyCfg::new("promote-consts"), - ]; - - let opt_coverage: &[&dyn MirPass<'tcx>] = - if tcx.sess.instrument_coverage() { &[&coverage::InstrumentCoverage] } else { &[] }; - - run_passes(tcx, &mut body, MirPhase::ConstPromotion, &[promote, opt_coverage]); + pm::run_passes( + tcx, + &mut body, + &[ + &promote_pass, + &simplify::SimplifyCfg::new("promote-consts"), + &coverage::InstrumentCoverage, + ], + ); let promoted = promote_pass.promoted_fragments.into_inner(); (tcx.alloc_steal_mir(body), tcx.alloc_steal_promoted(promoted)) @@ -388,19 +334,10 @@ fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) - // Technically we want to not run on regular const items, but oli-obk doesn't know how to // conveniently detect that at this point without looking at the HIR. hir::ConstContext::Const => { - #[rustfmt::skip] - let optimizations: &[&dyn MirPass<'_>] = &[ - &const_prop::ConstProp, - ]; - - #[rustfmt::skip] - run_passes( + pm::run_passes( tcx, &mut body, - MirPhase::Optimization, - &[ - optimizations, - ], + &[&const_prop::ConstProp, &marker::PhaseChange(MirPhase::Optimization)], ); } } @@ -436,7 +373,7 @@ fn mir_drops_elaborated_and_const_checked<'tcx>( let def = ty::WithOptConstParam::unknown(did); // Do not compute the mir call graph without said call graph actually being used. - if inline::is_enabled(tcx) { + if inline::Inline.is_enabled(&tcx.sess) { let _ = tcx.mir_inliner_callees(ty::InstanceDef::Item(def)); } } @@ -444,8 +381,24 @@ fn mir_drops_elaborated_and_const_checked<'tcx>( let (body, _) = tcx.mir_promoted(def); let mut body = body.steal(); + // IMPORTANT + pm::run_passes(tcx, &mut body, &[&remove_false_edges::RemoveFalseEdges]); + + // Do a little drop elaboration before const-checking if `const_precise_live_drops` is enabled. + if check_consts::post_drop_elaboration::checking_enabled(&ConstCx::new(tcx, &body)) { + pm::run_passes( + tcx, + &mut body, + &[ + &simplify::SimplifyCfg::new("remove-false-edges"), + &remove_uninit_drops::RemoveUninitDrops, + ], + ); + check_consts::post_drop_elaboration::check_live_drops(tcx, &body); // FIXME: make this a MIR lint + } + run_post_borrowck_cleanup_passes(tcx, &mut body); - check_consts::post_drop_elaboration::check_live_drops(tcx, &body); + assert!(body.phase == MirPhase::DropLowering); tcx.alloc_steal_mir(body) } @@ -455,7 +408,7 @@ fn run_post_borrowck_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tc let post_borrowck_cleanup: &[&dyn MirPass<'tcx>] = &[ // Remove all things only needed by analysis - &simplify_branches::SimplifyBranches::new("initial"), + &simplify_branches::SimplifyConstCondition::new("initial"), &remove_noop_landing_pads::RemoveNoopLandingPads, &cleanup_post_borrowck::CleanupNonCodegenStatements, &simplify::SimplifyCfg::new("early-opt"), @@ -479,95 +432,72 @@ fn run_post_borrowck_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tc &deaggregator::Deaggregator, ]; - run_passes(tcx, body, MirPhase::DropLowering, &[post_borrowck_cleanup]); + pm::run_passes(tcx, body, post_borrowck_cleanup); } fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let mir_opt_level = tcx.sess.mir_opt_level(); + fn o1<T>(x: T) -> WithMinOptLevel<T> { + WithMinOptLevel(1, x) + } // Lowering generator control-flow and variables has to happen before we do anything else // to them. We run some optimizations before that, because they may be harder to do on the state // machine than on MIR with async primitives. - let optimizations_with_generators: &[&dyn MirPass<'tcx>] = &[ - &reveal_all::RevealAll, // has to be done before inlining, since inlined code is in RevealAll mode. - &lower_slice_len::LowerSliceLenCalls, // has to be done before inlining, otherwise actual call will be almost always inlined. Also simple, so can just do first - &normalize_array_len::NormalizeArrayLen, // has to run after `slice::len` lowering - &unreachable_prop::UnreachablePropagation, - &uninhabited_enum_branching::UninhabitedEnumBranching, - &simplify::SimplifyCfg::new("after-uninhabited-enum-branching"), - &inline::Inline, - &generator::StateTransform, - ]; - - // Even if we don't do optimizations, we still have to lower generators for codegen. - let no_optimizations_with_generators: &[&dyn MirPass<'tcx>] = &[&generator::StateTransform]; - - // The main optimizations that we do on MIR. - let optimizations: &[&dyn MirPass<'tcx>] = &[ - &remove_storage_markers::RemoveStorageMarkers, - &remove_zsts::RemoveZsts, - &const_goto::ConstGoto, - &remove_unneeded_drops::RemoveUnneededDrops, - &match_branches::MatchBranchSimplification, - // inst combine is after MatchBranchSimplification to clean up Ne(_1, false) - &multiple_return_terminators::MultipleReturnTerminators, - &instcombine::InstCombine, - &separate_const_switch::SeparateConstSwitch, - &const_prop::ConstProp, - &simplify_branches::SimplifyBranches::new("after-const-prop"), - &early_otherwise_branch::EarlyOtherwiseBranch, - &simplify_comparison_integral::SimplifyComparisonIntegral, - &simplify_try::SimplifyArmIdentity, - &simplify_try::SimplifyBranchSame, - &dest_prop::DestinationPropagation, - &simplify_branches::SimplifyBranches::new("final"), - &remove_noop_landing_pads::RemoveNoopLandingPads, - &simplify::SimplifyCfg::new("final"), - &nrvo::RenameReturnPlace, - &const_debuginfo::ConstDebugInfo, - &simplify::SimplifyLocals, - &multiple_return_terminators::MultipleReturnTerminators, - &deduplicate_blocks::DeduplicateBlocks, - ]; - - // Optimizations to run even if mir optimizations have been disabled. - let no_optimizations: &[&dyn MirPass<'tcx>] = &[ - // FIXME(#70073): This pass is responsible for both optimization as well as some lints. - &const_prop::ConstProp, - ]; - - // Some cleanup necessary at least for LLVM and potentially other codegen backends. - let pre_codegen_cleanup: &[&dyn MirPass<'tcx>] = &[ - &add_call_guards::CriticalCallEdges, - // Dump the end result for testing and debugging purposes. - &dump_mir::Marker("PreCodegen"), - ]; - - // End of pass declarations, now actually run the passes. - // Generator Lowering - #[rustfmt::skip] - run_passes( + pm::run_passes( tcx, body, - MirPhase::GeneratorLowering, &[ - if mir_opt_level > 0 { - optimizations_with_generators - } else { - no_optimizations_with_generators - } + &reveal_all::RevealAll, // has to be done before inlining, since inlined code is in RevealAll mode. + &lower_slice_len::LowerSliceLenCalls, // has to be done before inlining, otherwise actual call will be almost always inlined. Also simple, so can just do first + &normalize_array_len::NormalizeArrayLen, // has to run after `slice::len` lowering + &unreachable_prop::UnreachablePropagation, + &uninhabited_enum_branching::UninhabitedEnumBranching, + &o1(simplify::SimplifyCfg::new("after-uninhabited-enum-branching")), + &inline::Inline, + &generator::StateTransform, ], ); - // Main optimization passes - #[rustfmt::skip] - run_passes( + assert!(body.phase == MirPhase::GeneratorLowering); + + // The main optimizations that we do on MIR. + pm::run_passes( tcx, body, - MirPhase::Optimization, &[ - if mir_opt_level > 0 { optimizations } else { no_optimizations }, - pre_codegen_cleanup, + &remove_storage_markers::RemoveStorageMarkers, + &remove_zsts::RemoveZsts, + &const_goto::ConstGoto, + &remove_unneeded_drops::RemoveUnneededDrops, + &match_branches::MatchBranchSimplification, + // inst combine is after MatchBranchSimplification to clean up Ne(_1, false) + &multiple_return_terminators::MultipleReturnTerminators, + &instcombine::InstCombine, + &separate_const_switch::SeparateConstSwitch, + // + // FIXME(#70073): This pass is responsible for both optimization as well as some lints. + &const_prop::ConstProp, + // + // Const-prop runs unconditionally, but doesn't mutate the MIR at mir-opt-level=0. + &o1(simplify_branches::SimplifyConstCondition::new("after-const-prop")), + &early_otherwise_branch::EarlyOtherwiseBranch, + &simplify_comparison_integral::SimplifyComparisonIntegral, + &simplify_try::SimplifyArmIdentity, + &simplify_try::SimplifyBranchSame, + &dest_prop::DestinationPropagation, + &o1(simplify_branches::SimplifyConstCondition::new("final")), + &o1(remove_noop_landing_pads::RemoveNoopLandingPads), + &o1(simplify::SimplifyCfg::new("final")), + &nrvo::RenameReturnPlace, + &const_debuginfo::ConstDebugInfo, + &simplify::SimplifyLocals, + &multiple_return_terminators::MultipleReturnTerminators, + &deduplicate_blocks::DeduplicateBlocks, + // Some cleanup necessary at least for LLVM and potentially other codegen backends. + &add_call_guards::CriticalCallEdges, + &marker::PhaseChange(MirPhase::Optimization), + // Dump the end result for testing and debugging purposes. + &dump_mir::Marker("PreCodegen"), ], ); } diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs index 5848163af72..4c4497ad629 100644 --- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs +++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs @@ -135,7 +135,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { } } -fn resolve_rust_intrinsic( +fn resolve_rust_intrinsic<'tcx>( tcx: TyCtxt<'tcx>, func_ty: Ty<'tcx>, ) -> Option<(Symbol, SubstsRef<'tcx>)> { @@ -148,7 +148,7 @@ fn resolve_rust_intrinsic( None } -fn validate_simd_shuffle(tcx: TyCtxt<'tcx>, args: &[Operand<'tcx>], span: Span) { +fn validate_simd_shuffle<'tcx>(tcx: TyCtxt<'tcx>, args: &[Operand<'tcx>], span: Span) { match &args[2] { Operand::Constant(_) => {} // all good _ => { diff --git a/compiler/rustc_mir_transform/src/lower_slice_len.rs b/compiler/rustc_mir_transform/src/lower_slice_len.rs index 822a372d8ce..c8297744873 100644 --- a/compiler/rustc_mir_transform/src/lower_slice_len.rs +++ b/compiler/rustc_mir_transform/src/lower_slice_len.rs @@ -10,6 +10,10 @@ use rustc_middle::ty::{self, TyCtxt}; pub struct LowerSliceLenCalls; impl<'tcx> MirPass<'tcx> for LowerSliceLenCalls { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.opts.mir_opt_level() > 0 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { lower_slice_len_calls(tcx, body) } diff --git a/compiler/rustc_mir_transform/src/marker.rs b/compiler/rustc_mir_transform/src/marker.rs new file mode 100644 index 00000000000..06819fc1d37 --- /dev/null +++ b/compiler/rustc_mir_transform/src/marker.rs @@ -0,0 +1,20 @@ +use std::borrow::Cow; + +use crate::MirPass; +use rustc_middle::mir::{Body, MirPhase}; +use rustc_middle::ty::TyCtxt; + +/// Changes the MIR phase without changing the MIR itself. +pub struct PhaseChange(pub MirPhase); + +impl<'tcx> MirPass<'tcx> for PhaseChange { + fn phase_change(&self) -> Option<MirPhase> { + Some(self.0) + } + + fn name(&self) -> Cow<'_, str> { + Cow::from(format!("PhaseChange-{:?}", self.0)) + } + + fn run_pass(&self, _: TyCtxt<'tcx>, _body: &mut Body<'tcx>) {} +} diff --git a/compiler/rustc_mir_transform/src/match_branches.rs b/compiler/rustc_mir_transform/src/match_branches.rs index c618abe9d05..3c14a324c36 100644 --- a/compiler/rustc_mir_transform/src/match_branches.rs +++ b/compiler/rustc_mir_transform/src/match_branches.rs @@ -40,11 +40,11 @@ pub struct MatchBranchSimplification; /// ``` impl<'tcx> MirPass<'tcx> for MatchBranchSimplification { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if tcx.sess.mir_opt_level() < 3 { - return; - } + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() >= 3 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let def_id = body.source.def_id(); let param_env = tcx.param_env(def_id); diff --git a/compiler/rustc_mir_transform/src/multiple_return_terminators.rs b/compiler/rustc_mir_transform/src/multiple_return_terminators.rs index b614917a883..22b6dead99c 100644 --- a/compiler/rustc_mir_transform/src/multiple_return_terminators.rs +++ b/compiler/rustc_mir_transform/src/multiple_return_terminators.rs @@ -9,11 +9,11 @@ use rustc_middle::ty::TyCtxt; pub struct MultipleReturnTerminators; impl<'tcx> MirPass<'tcx> for MultipleReturnTerminators { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if tcx.sess.mir_opt_level() < 4 { - return; - } + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() >= 4 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // find basic blocks with no statement and a return terminator let mut bbs_simple_returns = BitSet::new_empty(body.basic_blocks().len()); let def_id = body.source.def_id(); diff --git a/compiler/rustc_mir_transform/src/normalize_array_len.rs b/compiler/rustc_mir_transform/src/normalize_array_len.rs index a04a0b51531..e4ac57ac925 100644 --- a/compiler/rustc_mir_transform/src/normalize_array_len.rs +++ b/compiler/rustc_mir_transform/src/normalize_array_len.rs @@ -14,11 +14,11 @@ const MAX_NUM_LOCALS: usize = 3000; pub struct NormalizeArrayLen; impl<'tcx> MirPass<'tcx> for NormalizeArrayLen { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if tcx.sess.mir_opt_level() < 4 { - return; - } + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() >= 4 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // early returns for edge cases of highly unrolled functions if body.basic_blocks().len() > MAX_NUM_BLOCKS { return; @@ -85,7 +85,7 @@ struct Patcher<'a, 'tcx> { statement_idx: usize, } -impl<'a, 'tcx> Patcher<'a, 'tcx> { +impl<'tcx> Patcher<'_, 'tcx> { fn patch_expand_statement( &mut self, statement: &mut Statement<'tcx>, diff --git a/compiler/rustc_mir_transform/src/nrvo.rs b/compiler/rustc_mir_transform/src/nrvo.rs index 3ac4e77cf9a..797f7ee2685 100644 --- a/compiler/rustc_mir_transform/src/nrvo.rs +++ b/compiler/rustc_mir_transform/src/nrvo.rs @@ -33,11 +33,11 @@ use crate::MirPass; pub struct RenameReturnPlace; impl<'tcx> MirPass<'tcx> for RenameReturnPlace { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut mir::Body<'tcx>) { - if tcx.sess.mir_opt_level() == 0 { - return; - } + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() > 0 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut mir::Body<'tcx>) { let def_id = body.source.def_id(); let returned_local = match local_eligible_for_nrvo(body) { Some(l) => l, @@ -165,7 +165,7 @@ struct RenameToReturnPlace<'tcx> { } /// Replaces all uses of `self.to_rename` with `_0`. -impl MutVisitor<'tcx> for RenameToReturnPlace<'tcx> { +impl<'tcx> MutVisitor<'tcx> for RenameToReturnPlace<'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } @@ -221,7 +221,7 @@ impl IsReturnPlaceRead { } } -impl Visitor<'tcx> for IsReturnPlaceRead { +impl<'tcx> Visitor<'tcx> for IsReturnPlaceRead { fn visit_local(&mut self, &l: &Local, ctxt: PlaceContext, _: Location) { if l == mir::RETURN_PLACE && ctxt.is_use() && !ctxt.is_place_assignment() { self.0 = true; diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs new file mode 100644 index 00000000000..8725eae8709 --- /dev/null +++ b/compiler/rustc_mir_transform/src/pass_manager.rs @@ -0,0 +1,144 @@ +use std::borrow::Cow; + +use rustc_middle::mir::{self, Body, MirPhase}; +use rustc_middle::ty::TyCtxt; +use rustc_session::Session; + +use crate::{validate, MirPass}; + +/// Just like `MirPass`, except it cannot mutate `Body`. +pub trait MirLint<'tcx> { + fn name(&self) -> Cow<'_, str> { + let name = std::any::type_name::<Self>(); + if let Some(tail) = name.rfind(':') { + Cow::from(&name[tail + 1..]) + } else { + Cow::from(name) + } + } + + fn is_enabled(&self, _sess: &Session) -> bool { + true + } + + fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>); +} + +/// An adapter for `MirLint`s that implements `MirPass`. +#[derive(Debug, Clone)] +pub struct Lint<T>(pub T); + +impl<'tcx, T> MirPass<'tcx> for Lint<T> +where + T: MirLint<'tcx>, +{ + fn name(&self) -> Cow<'_, str> { + self.0.name() + } + + fn is_enabled(&self, sess: &Session) -> bool { + self.0.is_enabled(sess) + } + + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + self.0.run_lint(tcx, body) + } + + fn is_mir_dump_enabled(&self) -> bool { + false + } +} + +pub struct WithMinOptLevel<T>(pub u32, pub T); + +impl<'tcx, T> MirPass<'tcx> for WithMinOptLevel<T> +where + T: MirPass<'tcx>, +{ + fn name(&self) -> Cow<'_, str> { + self.1.name() + } + + fn is_enabled(&self, sess: &Session) -> bool { + sess.mir_opt_level() >= self.0 as usize + } + + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + self.1.run_pass(tcx, body) + } + + fn phase_change(&self) -> Option<MirPhase> { + self.1.phase_change() + } +} + +pub fn run_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, passes: &[&dyn MirPass<'tcx>]) { + let start_phase = body.phase; + let mut cnt = 0; + + let validate = tcx.sess.opts.debugging_opts.validate_mir; + + if validate { + validate_body(tcx, body, format!("start of phase transition from {:?}", start_phase)); + } + + for pass in passes { + if !pass.is_enabled(&tcx.sess) { + continue; + } + + let name = pass.name(); + let dump_enabled = pass.is_mir_dump_enabled(); + + if dump_enabled { + dump_mir(tcx, body, start_phase, &name, cnt, false); + } + + pass.run_pass(tcx, body); + + if dump_enabled { + dump_mir(tcx, body, start_phase, &name, cnt, true); + cnt += 1; + } + + if let Some(new_phase) = pass.phase_change() { + if body.phase >= new_phase { + panic!("Invalid MIR phase transition from {:?} to {:?}", body.phase, new_phase); + } + + body.phase = new_phase; + } + + if validate { + validate_body(tcx, body, format!("after pass {}", pass.name())); + } + } + + if validate || body.phase == MirPhase::Optimization { + validate_body(tcx, body, format!("end of phase transition to {:?}", body.phase)); + } +} + +pub fn validate_body<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, when: String) { + validate::Validator { when, mir_phase: body.phase }.run_pass(tcx, body); +} + +pub fn dump_mir<'tcx>( + tcx: TyCtxt<'tcx>, + body: &Body<'tcx>, + phase: MirPhase, + pass_name: &str, + cnt: usize, + is_after: bool, +) { + let phase_index = phase as u32; + + mir::dump_mir( + tcx, + Some(&format_args!("{:03}-{:03}", phase_index, cnt)), + pass_name, + if is_after { &"after" } else { &"before" }, + body, + |_, _| Ok(()), + ); +} diff --git a/compiler/rustc_mir_transform/src/remove_false_edges.rs b/compiler/rustc_mir_transform/src/remove_false_edges.rs new file mode 100644 index 00000000000..71f5ccf7e24 --- /dev/null +++ b/compiler/rustc_mir_transform/src/remove_false_edges.rs @@ -0,0 +1,29 @@ +use rustc_middle::mir::{Body, TerminatorKind}; +use rustc_middle::ty::TyCtxt; + +use crate::MirPass; + +/// Removes `FalseEdge` and `FalseUnwind` terminators from the MIR. +/// +/// These are only needed for borrow checking, and can be removed afterwards. +/// +/// FIXME: This should probably have its own MIR phase. +pub struct RemoveFalseEdges; + +impl<'tcx> MirPass<'tcx> for RemoveFalseEdges { + fn run_pass(&self, _: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + for block in body.basic_blocks_mut() { + let terminator = block.terminator_mut(); + terminator.kind = match terminator.kind { + TerminatorKind::FalseEdge { real_target, .. } => { + TerminatorKind::Goto { target: real_target } + } + TerminatorKind::FalseUnwind { real_target, .. } => { + TerminatorKind::Goto { target: real_target } + } + + _ => continue, + } + } + } +} diff --git a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs index 298bcd9dc24..2a73e341f16 100644 --- a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs +++ b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs @@ -10,18 +10,14 @@ use rustc_target::spec::PanicStrategy; /// code for these. pub struct RemoveNoopLandingPads; -pub fn remove_noop_landing_pads<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if tcx.sess.panic_strategy() == PanicStrategy::Abort { - return; +impl<'tcx> MirPass<'tcx> for RemoveNoopLandingPads { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.panic_strategy() != PanicStrategy::Abort } - debug!("remove_noop_landing_pads({:?})", body); - - RemoveNoopLandingPads.remove_nop_landing_pads(body) -} -impl<'tcx> MirPass<'tcx> for RemoveNoopLandingPads { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - remove_noop_landing_pads(tcx, body); + fn run_pass(&self, _: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + debug!("remove_noop_landing_pads({:?})", body); + self.remove_nop_landing_pads(body) } } diff --git a/compiler/rustc_mir_transform/src/remove_storage_markers.rs b/compiler/rustc_mir_transform/src/remove_storage_markers.rs index 0c7323cbac5..c9b6e1459d3 100644 --- a/compiler/rustc_mir_transform/src/remove_storage_markers.rs +++ b/compiler/rustc_mir_transform/src/remove_storage_markers.rs @@ -7,6 +7,10 @@ use rustc_middle::ty::TyCtxt; pub struct RemoveStorageMarkers; impl<'tcx> MirPass<'tcx> for RemoveStorageMarkers { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() > 0 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { if tcx.sess.emit_lifetime_markers() { return; diff --git a/compiler/rustc_mir_transform/src/remove_uninit_drops.rs b/compiler/rustc_mir_transform/src/remove_uninit_drops.rs new file mode 100644 index 00000000000..fc5ac97e3e1 --- /dev/null +++ b/compiler/rustc_mir_transform/src/remove_uninit_drops.rs @@ -0,0 +1,171 @@ +use rustc_index::bit_set::BitSet; +use rustc_middle::mir::{Body, Field, Rvalue, Statement, StatementKind, TerminatorKind}; +use rustc_middle::ty::subst::SubstsRef; +use rustc_middle::ty::{self, ParamEnv, Ty, TyCtxt, VariantDef}; +use rustc_mir_dataflow::impls::MaybeInitializedPlaces; +use rustc_mir_dataflow::move_paths::{LookupResult, MoveData, MovePathIndex}; +use rustc_mir_dataflow::{self, move_path_children_matching, Analysis, MoveDataParamEnv}; + +use crate::MirPass; + +/// Removes `Drop` and `DropAndReplace` terminators whose target is known to be uninitialized at +/// that point. +/// +/// This is redundant with drop elaboration, but we need to do it prior to const-checking, and +/// running const-checking after drop elaboration makes it opimization dependent, causing issues +/// like [#90770]. +/// +/// [#90770]: https://github.com/rust-lang/rust/issues/90770 +pub struct RemoveUninitDrops; + +impl<'tcx> MirPass<'tcx> for RemoveUninitDrops { + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let param_env = tcx.param_env(body.source.def_id()); + let Ok(move_data) = MoveData::gather_moves(body, tcx, param_env) else { + // We could continue if there are move errors, but there's not much point since our + // init data isn't complete. + return; + }; + + let mdpe = MoveDataParamEnv { move_data, param_env }; + let mut maybe_inits = MaybeInitializedPlaces::new(tcx, body, &mdpe) + .into_engine(tcx, body) + .pass_name("remove_uninit_drops") + .iterate_to_fixpoint() + .into_results_cursor(body); + + let mut to_remove = vec![]; + for (bb, block) in body.basic_blocks().iter_enumerated() { + let terminator = block.terminator(); + let (TerminatorKind::Drop { place, .. } | TerminatorKind::DropAndReplace { place, .. }) + = &terminator.kind + else { continue }; + + maybe_inits.seek_before_primary_effect(body.terminator_loc(bb)); + + // If there's no move path for the dropped place, it's probably a `Deref`. Let it alone. + let LookupResult::Exact(mpi) = mdpe.move_data.rev_lookup.find(place.as_ref()) else { + continue; + }; + + let should_keep = is_needs_drop_and_init( + tcx, + param_env, + maybe_inits.get(), + &mdpe.move_data, + place.ty(body, tcx).ty, + mpi, + ); + if !should_keep { + to_remove.push(bb) + } + } + + for bb in to_remove { + let block = &mut body.basic_blocks_mut()[bb]; + + let (TerminatorKind::Drop { target, .. } | TerminatorKind::DropAndReplace { target, .. }) + = &block.terminator().kind + else { unreachable!() }; + + // Replace block terminator with `Goto`. + let target = *target; + let old_terminator_kind = std::mem::replace( + &mut block.terminator_mut().kind, + TerminatorKind::Goto { target }, + ); + + // If this is a `DropAndReplace`, we need to emulate the assignment to the return place. + if let TerminatorKind::DropAndReplace { place, value, .. } = old_terminator_kind { + block.statements.push(Statement { + source_info: block.terminator().source_info, + kind: StatementKind::Assign(Box::new((place, Rvalue::Use(value)))), + }); + } + } + } +} + +fn is_needs_drop_and_init<'tcx>( + tcx: TyCtxt<'tcx>, + param_env: ParamEnv<'tcx>, + maybe_inits: &BitSet<MovePathIndex>, + move_data: &MoveData<'tcx>, + ty: Ty<'tcx>, + mpi: MovePathIndex, +) -> bool { + // No need to look deeper if the root is definitely uninit or if it has no `Drop` impl. + if !maybe_inits.contains(mpi) || !ty.needs_drop(tcx, param_env) { + return false; + } + + let field_needs_drop_and_init = |(f, f_ty, mpi)| { + let child = move_path_children_matching(move_data, mpi, |x| x.is_field_to(f)); + let Some(mpi) = child else { + return f_ty.needs_drop(tcx, param_env); + }; + + is_needs_drop_and_init(tcx, param_env, maybe_inits, move_data, f_ty, mpi) + }; + + // This pass is only needed for const-checking, so it doesn't handle as many cases as + // `DropCtxt::open_drop`, since they aren't relevant in a const-context. + match ty.kind() { + ty::Adt(adt, substs) => { + let dont_elaborate = adt.is_union() || adt.is_manually_drop() || adt.has_dtor(tcx); + if dont_elaborate { + return true; + } + + // Look at all our fields, or if we are an enum all our variants and their fields. + // + // If a field's projection *is not* present in `MoveData`, it has the same + // initializedness as its parent (maybe init). + // + // If its projection *is* present in `MoveData`, then the field may have been moved + // from separate from its parent. Recurse. + adt.variants.iter_enumerated().any(|(vid, variant)| { + // Enums have multiple variants, which are discriminated with a `Downcast` projection. + // Structs have a single variant, and don't use a `Downcast` projection. + let mpi = if adt.is_enum() { + let downcast = + move_path_children_matching(move_data, mpi, |x| x.is_downcast_to(vid)); + let Some(dc_mpi) = downcast else { + return variant_needs_drop(tcx, param_env, substs, variant); + }; + + dc_mpi + } else { + mpi + }; + + variant + .fields + .iter() + .enumerate() + .map(|(f, field)| (Field::from_usize(f), field.ty(tcx, substs), mpi)) + .any(field_needs_drop_and_init) + }) + } + + ty::Tuple(_) => ty + .tuple_fields() + .enumerate() + .map(|(f, f_ty)| (Field::from_usize(f), f_ty, mpi)) + .any(field_needs_drop_and_init), + + _ => true, + } +} + +fn variant_needs_drop<'tcx>( + tcx: TyCtxt<'tcx>, + param_env: ParamEnv<'tcx>, + substs: SubstsRef<'tcx>, + variant: &VariantDef, +) -> bool { + variant.fields.iter().any(|field| { + let f_ty = field.ty(tcx, substs); + f_ty.needs_drop(tcx, param_env) + }) +} diff --git a/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs b/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs index c71bc512c31..39f78e9555e 100644 --- a/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs +++ b/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs @@ -1,4 +1,8 @@ -//! This pass replaces a drop of a type that does not need dropping, with a goto +//! This pass replaces a drop of a type that does not need dropping, with a goto. +//! +//! When the MIR is built, we check `needs_drop` before emitting a `Drop` for a place. This pass is +//! useful because (unlike MIR building) it runs after type checking, so it can make use of +//! `Reveal::All` to provide more precies type information. use crate::MirPass; use rustc_middle::mir::*; diff --git a/compiler/rustc_mir_transform/src/remove_zsts.rs b/compiler/rustc_mir_transform/src/remove_zsts.rs index d93ffa38c69..1d912e61409 100644 --- a/compiler/rustc_mir_transform/src/remove_zsts.rs +++ b/compiler/rustc_mir_transform/src/remove_zsts.rs @@ -8,6 +8,10 @@ use rustc_middle::ty::{self, Ty, TyCtxt}; pub struct RemoveZsts; impl<'tcx> MirPass<'tcx> for RemoveZsts { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() > 0 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // Avoid query cycles (generators require optimized MIR for layout). if tcx.type_of(body.source.def_id()).is_generator() { diff --git a/compiler/rustc_mir_transform/src/required_consts.rs b/compiler/rustc_mir_transform/src/required_consts.rs index 8b64ad65ab3..80c87cafea1 100644 --- a/compiler/rustc_mir_transform/src/required_consts.rs +++ b/compiler/rustc_mir_transform/src/required_consts.rs @@ -12,7 +12,7 @@ impl<'a, 'tcx> RequiredConstsVisitor<'a, 'tcx> { } } -impl<'a, 'tcx> Visitor<'tcx> for RequiredConstsVisitor<'a, 'tcx> { +impl<'tcx> Visitor<'tcx> for RequiredConstsVisitor<'_, 'tcx> { fn visit_constant(&mut self, constant: &Constant<'tcx>, _: Location) { if let Some(ct) = constant.literal.const_for_ty() { if let ConstKind::Unevaluated(_) = ct.val { diff --git a/compiler/rustc_mir_transform/src/reveal_all.rs b/compiler/rustc_mir_transform/src/reveal_all.rs index 3bcb71b64f4..ee661793a44 100644 --- a/compiler/rustc_mir_transform/src/reveal_all.rs +++ b/compiler/rustc_mir_transform/src/reveal_all.rs @@ -8,15 +8,18 @@ use rustc_middle::ty::{self, Ty, TyCtxt}; pub struct RevealAll; impl<'tcx> MirPass<'tcx> for RevealAll { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.opts.mir_opt_level() >= 3 || super::inline::Inline.is_enabled(sess) + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - // This pass must run before inlining, since we insert callee bodies in RevealAll mode. // Do not apply this transformation to generators. - if (tcx.sess.mir_opt_level() >= 3 || super::inline::is_enabled(tcx)) - && body.generator.is_none() - { - let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); - RevealAllVisitor { tcx, param_env }.visit_body(body); + if body.generator.is_some() { + return; } + + let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); + RevealAllVisitor { tcx, param_env }.visit_body(body); } } @@ -33,6 +36,9 @@ impl<'tcx> MutVisitor<'tcx> for RevealAllVisitor<'tcx> { #[inline] fn visit_ty(&mut self, ty: &mut Ty<'tcx>, _: TyContext) { - *ty = self.tcx.normalize_erasing_regions(self.param_env, ty); + // We have to use `try_normalize_erasing_regions` here, since it's + // possible that we visit impossible-to-satisfy where clauses here, + // see #91745 + *ty = self.tcx.try_normalize_erasing_regions(self.param_env, *ty).unwrap_or(ty); } } diff --git a/compiler/rustc_mir_transform/src/separate_const_switch.rs b/compiler/rustc_mir_transform/src/separate_const_switch.rs index 3002e7041b0..612fce71f91 100644 --- a/compiler/rustc_mir_transform/src/separate_const_switch.rs +++ b/compiler/rustc_mir_transform/src/separate_const_switch.rs @@ -45,11 +45,11 @@ use smallvec::SmallVec; pub struct SeparateConstSwitch; impl<'tcx> MirPass<'tcx> for SeparateConstSwitch { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if tcx.sess.mir_opt_level() < 4 { - return; - } + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() >= 4 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // If execution did something, applying a simplification layer // helps later passes optimize the copy away. if separate_const_switch(body) > 0 { @@ -59,7 +59,7 @@ impl<'tcx> MirPass<'tcx> for SeparateConstSwitch { } /// Returns the amount of blocks that were duplicated -pub fn separate_const_switch<'tcx>(body: &mut Body<'tcx>) -> usize { +pub fn separate_const_switch(body: &mut Body<'_>) -> usize { let mut new_blocks: SmallVec<[(BasicBlock, BasicBlock); 6]> = SmallVec::new(); let predecessors = body.predecessors(); 'block_iter: for (block_id, block) in body.basic_blocks().iter_enumerated() { diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index f59aaa664f3..58996dcd673 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -17,8 +17,8 @@ use std::iter; use crate::util::expand_aggregate; use crate::{ - abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, remove_noop_landing_pads, - run_passes, simplify, + abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, marker, pass_manager as pm, + remove_noop_landing_pads, simplify, }; use rustc_middle::mir::patch::MirPatch; use rustc_mir_dataflow::elaborate_drops::{self, DropElaborator, DropFlagMode, DropStyle}; @@ -64,7 +64,19 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<' build_call_shim(tcx, instance, Some(Adjustment::RefMut), CallKind::Direct(call_mut)) } - ty::InstanceDef::DropGlue(def_id, ty) => build_drop_shim(tcx, def_id, ty), + + ty::InstanceDef::DropGlue(def_id, ty) => { + // FIXME(#91576): Drop shims for generators aren't subject to the MIR passes at the end + // of this function. Is this intentional? + if let Some(ty::Generator(gen_def_id, substs, _)) = ty.map(ty::TyS::kind) { + let body = tcx.optimized_mir(*gen_def_id).generator_drop().unwrap(); + let body = body.clone().subst(tcx, substs); + debug!("make_shim({:?}) = {:?}", instance, body); + return body; + } + + build_drop_shim(tcx, def_id, ty) + } ty::InstanceDef::CloneShim(def_id, ty) => build_clone_shim(tcx, def_id, ty), ty::InstanceDef::Virtual(..) => { bug!("InstanceDef::Virtual ({:?}) is for direct calls only", instance) @@ -75,17 +87,17 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<' }; debug!("make_shim({:?}) = untransformed {:?}", instance, result); - run_passes( + pm::run_passes( tcx, &mut result, - MirPhase::Const, - &[&[ + &[ &add_moves_for_packed_drops::AddMovesForPackedDrops, &remove_noop_landing_pads::RemoveNoopLandingPads, &simplify::SimplifyCfg::new("make_shim"), &add_call_guards::CriticalCallEdges, &abort_unwinding_calls::AbortUnwindingCalls, - ]], + &marker::PhaseChange(MirPhase::Const), + ], ); debug!("make_shim({:?}) = {:?}", instance, result); @@ -132,11 +144,7 @@ fn local_decls_for_sig<'tcx>( fn build_drop_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, ty: Option<Ty<'tcx>>) -> Body<'tcx> { debug!("build_drop_shim(def_id={:?}, ty={:?})", def_id, ty); - // Check if this is a generator, if so, return the drop glue for it - if let Some(&ty::Generator(gen_def_id, substs, _)) = ty.map(|ty| ty.kind()) { - let body = tcx.optimized_mir(gen_def_id).generator_drop().unwrap(); - return body.clone().subst(tcx, substs); - } + assert!(!matches!(ty, Some(ty) if ty.is_generator())); let substs = if let Some(ty) = ty { tcx.intern_substs(&[ty.into()]) @@ -239,7 +247,7 @@ pub struct DropShimElaborator<'a, 'tcx> { pub param_env: ty::ParamEnv<'tcx>, } -impl<'a, 'tcx> fmt::Debug for DropShimElaborator<'a, 'tcx> { +impl fmt::Debug for DropShimElaborator<'_, '_> { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { Ok(()) } @@ -329,7 +337,7 @@ struct CloneShimBuilder<'tcx> { sig: ty::FnSig<'tcx>, } -impl CloneShimBuilder<'tcx> { +impl<'tcx> CloneShimBuilder<'tcx> { fn new(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Self { // we must subst the self_ty because it's // otherwise going to be TySelf and we can't index @@ -769,7 +777,7 @@ pub fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> Body<'_> { adt_def.variants[variant_index].fields.iter().enumerate().map(|(idx, field_def)| { (Operand::Move(Place::from(Local::new(idx + 1))), field_def.ty(tcx, substs)) }), - AggregateKind::Adt(adt_def, variant_index, substs, None, None), + AggregateKind::Adt(adt_def.did, variant_index, substs, None, None), source_info, tcx, ) diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index d6cd505cbb5..7992124bacd 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -47,7 +47,7 @@ impl SimplifyCfg { } } -pub fn simplify_cfg(tcx: TyCtxt<'tcx>, body: &mut Body<'_>) { +pub fn simplify_cfg<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { CfgSimplifier::new(body).simplify(); remove_dead_blocks(tcx, body); @@ -262,7 +262,7 @@ impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> { } } -pub fn remove_dead_blocks(tcx: TyCtxt<'tcx>, body: &mut Body<'_>) { +pub fn remove_dead_blocks<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let reachable = traversal::reachable_as_bitset(body); let num_blocks = body.basic_blocks().len(); if num_blocks == reachable.count() { @@ -368,6 +368,10 @@ fn save_unreachable_coverage( pub struct SimplifyLocals; impl<'tcx> MirPass<'tcx> for SimplifyLocals { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() > 0 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { trace!("running SimplifyLocals on {:?}", body.source); simplify_locals(body, tcx); @@ -450,7 +454,7 @@ impl UsedLocals { } /// Updates the use counts to reflect the removal of given statement. - fn statement_removed(&mut self, statement: &Statement<'tcx>) { + fn statement_removed(&mut self, statement: &Statement<'_>) { self.increment = false; // The location of the statement is irrelevant. @@ -459,7 +463,7 @@ impl UsedLocals { } /// Visits a left-hand side of an assignment. - fn visit_lhs(&mut self, place: &Place<'tcx>, location: Location) { + fn visit_lhs(&mut self, place: &Place<'_>, location: Location) { if place.is_indirect() { // A use, not a definition. self.visit_place(place, PlaceContext::MutatingUse(MutatingUseContext::Store), location); @@ -476,7 +480,7 @@ impl UsedLocals { } } -impl Visitor<'_> for UsedLocals { +impl<'tcx> Visitor<'tcx> for UsedLocals { fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { match statement.kind { StatementKind::LlvmInlineAsm(..) @@ -514,7 +518,7 @@ impl Visitor<'_> for UsedLocals { } /// Removes unused definitions. Updates the used locals to reflect the changes made. -fn remove_unused_definitions<'a, 'tcx>(used_locals: &'a mut UsedLocals, body: &mut Body<'tcx>) { +fn remove_unused_definitions(used_locals: &mut UsedLocals, body: &mut Body<'_>) { // The use counts are updated as we remove the statements. A local might become unused // during the retain operation, leading to a temporary inconsistency (storage statements or // definitions referencing the local might remain). For correctness it is crucial that this diff --git a/compiler/rustc_mir_transform/src/simplify_branches.rs b/compiler/rustc_mir_transform/src/simplify_branches.rs index df90cfa318d..3bbae5b8976 100644 --- a/compiler/rustc_mir_transform/src/simplify_branches.rs +++ b/compiler/rustc_mir_transform/src/simplify_branches.rs @@ -1,22 +1,21 @@ -//! A pass that simplifies branches when their condition is known. - use crate::MirPass; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; use std::borrow::Cow; -pub struct SimplifyBranches { +/// A pass that replaces a branch with a goto when its condition is known. +pub struct SimplifyConstCondition { label: String, } -impl SimplifyBranches { +impl SimplifyConstCondition { pub fn new(label: &str) -> Self { - SimplifyBranches { label: format!("SimplifyBranches-{}", label) } + SimplifyConstCondition { label: format!("SimplifyConstCondition-{}", label) } } } -impl<'tcx> MirPass<'tcx> for SimplifyBranches { +impl<'tcx> MirPass<'tcx> for SimplifyConstCondition { fn name(&self) -> Cow<'_, str> { Cow::Borrowed(&self.label) } @@ -34,15 +33,8 @@ impl<'tcx> MirPass<'tcx> for SimplifyBranches { } => { let constant = c.literal.try_eval_bits(tcx, param_env, switch_ty); if let Some(constant) = constant { - let otherwise = targets.otherwise(); - let mut ret = TerminatorKind::Goto { target: otherwise }; - for (v, t) in targets.iter() { - if v == constant { - ret = TerminatorKind::Goto { target: t }; - break; - } - } - ret + let target = targets.target_for_value(constant); + TerminatorKind::Goto { target } } else { continue; } @@ -53,12 +45,6 @@ impl<'tcx> MirPass<'tcx> for SimplifyBranches { Some(v) if v == expected => TerminatorKind::Goto { target }, _ => continue, }, - TerminatorKind::FalseEdge { real_target, .. } => { - TerminatorKind::Goto { target: real_target } - } - TerminatorKind::FalseUnwind { real_target, .. } => { - TerminatorKind::Goto { target: real_target } - } _ => continue, }; } diff --git a/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs b/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs index 948fcd9f455..da683a33651 100644 --- a/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs +++ b/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs @@ -26,6 +26,10 @@ use rustc_middle::{ pub struct SimplifyComparisonIntegral; impl<'tcx> MirPass<'tcx> for SimplifyComparisonIntegral { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() > 0 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { trace!("Running SimplifyComparisonIntegral on {:?}", body.source); @@ -144,7 +148,7 @@ struct OptimizationFinder<'a, 'tcx> { body: &'a Body<'tcx>, } -impl<'a, 'tcx> OptimizationFinder<'a, 'tcx> { +impl<'tcx> OptimizationFinder<'_, 'tcx> { fn find_optimizations(&self) -> Vec<OptimizationInfo<'tcx>> { self.body .basic_blocks() diff --git a/compiler/rustc_mir_transform/src/simplify_try.rs b/compiler/rustc_mir_transform/src/simplify_try.rs index e436d73226a..7761d4006d3 100644 --- a/compiler/rustc_mir_transform/src/simplify_try.rs +++ b/compiler/rustc_mir_transform/src/simplify_try.rs @@ -102,7 +102,7 @@ fn get_arm_identity_info<'a, 'tcx>( type StmtIter<'a, 'tcx> = Peekable<Enumerate<Iter<'a, Statement<'tcx>>>>; - fn is_storage_stmt<'tcx>(stmt: &Statement<'tcx>) -> bool { + fn is_storage_stmt(stmt: &Statement<'_>) -> bool { matches!(stmt.kind, StatementKind::StorageLive(_) | StatementKind::StorageDead(_)) } @@ -122,8 +122,8 @@ fn get_arm_identity_info<'a, 'tcx>( /// Eats consecutive `StorageLive` and `StorageDead` Statements. /// The iterator `stmt_iter` is not advanced if none were found. - fn try_eat_storage_stmts<'a, 'tcx>( - stmt_iter: &mut StmtIter<'a, 'tcx>, + fn try_eat_storage_stmts( + stmt_iter: &mut StmtIter<'_, '_>, storage_live_stmts: &mut Vec<(usize, Local)>, storage_dead_stmts: &mut Vec<(usize, Local)>, ) { @@ -136,7 +136,7 @@ fn get_arm_identity_info<'a, 'tcx>( }) } - fn is_tmp_storage_stmt<'tcx>(stmt: &Statement<'tcx>) -> bool { + fn is_tmp_storage_stmt(stmt: &Statement<'_>) -> bool { use rustc_middle::mir::StatementKind::Assign; if let Assign(box (place, Rvalue::Use(Operand::Copy(p) | Operand::Move(p)))) = &stmt.kind { place.as_local().is_some() && p.as_local().is_some() @@ -147,8 +147,8 @@ fn get_arm_identity_info<'a, 'tcx>( /// Eats consecutive `Assign` Statements. // The iterator `stmt_iter` is not advanced if none were found. - fn try_eat_assign_tmp_stmts<'a, 'tcx>( - stmt_iter: &mut StmtIter<'a, 'tcx>, + fn try_eat_assign_tmp_stmts( + stmt_iter: &mut StmtIter<'_, '_>, tmp_assigns: &mut Vec<(Local, Local)>, nop_stmts: &mut Vec<usize>, ) { @@ -163,9 +163,9 @@ fn get_arm_identity_info<'a, 'tcx>( }) } - fn find_storage_live_dead_stmts_for_local<'tcx>( + fn find_storage_live_dead_stmts_for_local( local: Local, - stmts: &[Statement<'tcx>], + stmts: &[Statement<'_>], ) -> Option<(usize, usize)> { trace!("looking for {:?}", local); let mut storage_live_stmt = None; @@ -452,14 +452,14 @@ struct LocalUseCounter { } impl LocalUseCounter { - fn get_local_uses<'tcx>(body: &Body<'tcx>) -> IndexVec<Local, usize> { + fn get_local_uses(body: &Body<'_>) -> IndexVec<Local, usize> { let mut counter = LocalUseCounter { local_uses: IndexVec::from_elem(0, &body.local_decls) }; counter.visit_body(body); counter.local_uses } } -impl<'tcx> Visitor<'tcx> for LocalUseCounter { +impl Visitor<'_> for LocalUseCounter { fn visit_local(&mut self, local: &Local, context: PlaceContext, _location: Location) { if context.is_storage_marker() || context == PlaceContext::NonUse(NonUseContext::VarDebugInfo) @@ -510,7 +510,7 @@ fn match_set_variant_field<'tcx>(stmt: &Statement<'tcx>) -> Option<(Local, Local /// ```rust /// discriminant(_LOCAL_TO_SET) = VAR_IDX; /// ``` -fn match_set_discr<'tcx>(stmt: &Statement<'tcx>) -> Option<(Local, VariantIdx)> { +fn match_set_discr(stmt: &Statement<'_>) -> Option<(Local, VariantIdx)> { match &stmt.kind { StatementKind::SetDiscriminant { place, variant_index } => { Some((place.as_local()?, *variant_index)) @@ -588,7 +588,7 @@ struct SimplifyBranchSameOptimizationFinder<'a, 'tcx> { tcx: TyCtxt<'tcx>, } -impl<'a, 'tcx> SimplifyBranchSameOptimizationFinder<'a, 'tcx> { +impl<'tcx> SimplifyBranchSameOptimizationFinder<'_, 'tcx> { fn find(&self) -> Vec<SimplifyBranchSameOptimization> { self.body .basic_blocks() diff --git a/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs b/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs index 2aa50611290..77bc209539b 100644 --- a/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs +++ b/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs @@ -70,6 +70,10 @@ fn variant_discriminants<'tcx>( } impl<'tcx> MirPass<'tcx> for UninhabitedEnumBranching { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() > 0 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { if body.source.promoted.is_some() { return; diff --git a/compiler/rustc_mir_transform/src/unreachable_prop.rs b/compiler/rustc_mir_transform/src/unreachable_prop.rs index 64cd6f56a9f..9e755ab141a 100644 --- a/compiler/rustc_mir_transform/src/unreachable_prop.rs +++ b/compiler/rustc_mir_transform/src/unreachable_prop.rs @@ -11,13 +11,13 @@ use rustc_middle::ty::TyCtxt; pub struct UnreachablePropagation; impl MirPass<'_> for UnreachablePropagation { - fn run_pass<'tcx>(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if tcx.sess.mir_opt_level() < 4 { - // Enable only under -Zmir-opt-level=4 as in some cases (check the deeply-nested-opt - // perf benchmark) LLVM may spend quite a lot of time optimizing the generated code. - return; - } + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + // Enable only under -Zmir-opt-level=4 as in some cases (check the deeply-nested-opt + // perf benchmark) LLVM may spend quite a lot of time optimizing the generated code. + sess.mir_opt_level() >= 4 + } + fn run_pass<'tcx>(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let mut unreachable_blocks = FxHashSet::default(); let mut replacements = FxHashMap::default(); @@ -64,7 +64,7 @@ impl MirPass<'_> for UnreachablePropagation { } } -fn remove_successors<F>( +fn remove_successors<'tcx, F>( terminator_kind: &TerminatorKind<'tcx>, predicate: F, ) -> Option<TerminatorKind<'tcx>> |