diff options
Diffstat (limited to 'compiler/rustc_mir_transform/src')
47 files changed, 1194 insertions, 1183 deletions
diff --git a/compiler/rustc_mir_transform/src/add_retag.rs b/compiler/rustc_mir_transform/src/add_retag.rs index 7a8dee09c29..28a5a22dd9d 100644 --- a/compiler/rustc_mir_transform/src/add_retag.rs +++ b/compiler/rustc_mir_transform/src/add_retag.rs @@ -34,7 +34,7 @@ fn is_stable(place: PlaceRef<'_>) -> bool { } /// Determine whether this type may be a reference (or box), and thus needs retagging. -fn may_be_reference(ty: Ty<'tcx>) -> bool { +fn may_be_reference(ty: Ty<'_>) -> bool { match ty.kind() { // Primitive types that are not references ty::Bool @@ -58,11 +58,11 @@ fn may_be_reference(ty: Ty<'tcx>) -> bool { } impl<'tcx> MirPass<'tcx> for AddRetag { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if !tcx.sess.opts.debugging_opts.mir_emit_retag { - return; - } + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.opts.debugging_opts.mir_emit_retag + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // We need an `AllCallEdges` pass before we can do any work. super::add_call_guards::AllCallEdges.run_pass(tcx, body); diff --git a/compiler/rustc_mir_transform/src/check_const_item_mutation.rs b/compiler/rustc_mir_transform/src/check_const_item_mutation.rs index 27fe80a456f..a19a3c8b1d5 100644 --- a/compiler/rustc_mir_transform/src/check_const_item_mutation.rs +++ b/compiler/rustc_mir_transform/src/check_const_item_mutation.rs @@ -6,12 +6,12 @@ use rustc_middle::ty::TyCtxt; use rustc_session::lint::builtin::CONST_ITEM_MUTATION; use rustc_span::def_id::DefId; -use crate::MirPass; +use crate::MirLint; pub struct CheckConstItemMutation; -impl<'tcx> MirPass<'tcx> for CheckConstItemMutation { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { +impl<'tcx> MirLint<'tcx> for CheckConstItemMutation { + fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) { let mut checker = ConstMutationChecker { body, tcx, target_local: None }; checker.visit_body(&body); } @@ -23,7 +23,7 @@ struct ConstMutationChecker<'a, 'tcx> { target_local: Option<Local>, } -impl<'a, 'tcx> ConstMutationChecker<'a, 'tcx> { +impl<'tcx> ConstMutationChecker<'_, 'tcx> { fn is_const_item(&self, local: Local) -> Option<DefId> { if let Some(box LocalInfo::ConstRef { def_id }) = self.body.local_decls[local].local_info { Some(def_id) @@ -95,7 +95,7 @@ impl<'a, 'tcx> ConstMutationChecker<'a, 'tcx> { } } -impl<'a, 'tcx> Visitor<'tcx> for ConstMutationChecker<'a, 'tcx> { +impl<'tcx> Visitor<'tcx> for ConstMutationChecker<'_, 'tcx> { fn visit_statement(&mut self, stmt: &Statement<'tcx>, loc: Location) { if let StatementKind::Assign(box (lhs, _)) = &stmt.kind { // Check for assignment to fields of a constant diff --git a/compiler/rustc_mir_transform/src/check_packed_ref.rs b/compiler/rustc_mir_transform/src/check_packed_ref.rs index 49be34c7a28..23d59c80071 100644 --- a/compiler/rustc_mir_transform/src/check_packed_ref.rs +++ b/compiler/rustc_mir_transform/src/check_packed_ref.rs @@ -7,7 +7,7 @@ use rustc_session::lint::builtin::UNALIGNED_REFERENCES; use rustc_span::symbol::sym; use crate::util; -use crate::MirPass; +use crate::MirLint; pub(crate) fn provide(providers: &mut Providers) { *providers = Providers { unsafe_derive_on_repr_packed, ..*providers }; @@ -15,8 +15,8 @@ pub(crate) fn provide(providers: &mut Providers) { pub struct CheckPackedRef; -impl<'tcx> MirPass<'tcx> for CheckPackedRef { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { +impl<'tcx> MirLint<'tcx> for CheckPackedRef { + fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) { let param_env = tcx.param_env(body.source.def_id()); let source_info = SourceInfo::outermost(body.span); let mut checker = PackedRefChecker { body, tcx, param_env, source_info }; @@ -66,7 +66,7 @@ fn builtin_derive_def_id(tcx: TyCtxt<'_>, def_id: DefId) -> Option<DefId> { } } -impl<'a, 'tcx> Visitor<'tcx> for PackedRefChecker<'a, 'tcx> { +impl<'tcx> Visitor<'tcx> for PackedRefChecker<'_, 'tcx> { fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) { // Make sure we know where in the MIR we are. self.source_info = terminator.source_info; @@ -105,6 +105,11 @@ impl<'a, 'tcx> Visitor<'tcx> for PackedRefChecker<'a, 'tcx> { a misaligned reference is undefined behavior (even if that \ reference is never dereferenced)", ) + .help( + "copy the field contents to a local variable, or replace the \ + reference with a raw pointer and use `read_unaligned`/`write_unaligned` \ + (loads and stores via `*p` must be properly aligned even when using raw pointers)" + ) .emit() }, ); diff --git a/compiler/rustc_mir_transform/src/check_unsafety.rs b/compiler/rustc_mir_transform/src/check_unsafety.rs index 1ff9bd15721..fd93744d400 100644 --- a/compiler/rustc_mir_transform/src/check_unsafety.rs +++ b/compiler/rustc_mir_transform/src/check_unsafety.rs @@ -46,7 +46,7 @@ impl<'a, 'tcx> UnsafetyChecker<'a, 'tcx> { } } -impl<'a, 'tcx> Visitor<'tcx> for UnsafetyChecker<'a, 'tcx> { +impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> { fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) { self.source_info = terminator.source_info; match terminator.kind { @@ -104,10 +104,6 @@ impl<'a, 'tcx> Visitor<'tcx> for UnsafetyChecker<'a, 'tcx> { // safe (at least as emitted during MIR construction) } - StatementKind::LlvmInlineAsm { .. } => self.require_unsafe( - UnsafetyViolationKind::General, - UnsafetyViolationDetails::UseOfInlineAssembly, - ), StatementKind::CopyNonOverlapping(..) => unreachable!(), } self.super_statement(statement, location); @@ -117,8 +113,8 @@ impl<'a, 'tcx> Visitor<'tcx> for UnsafetyChecker<'a, 'tcx> { match rvalue { Rvalue::Aggregate(box ref aggregate, _) => match aggregate { &AggregateKind::Array(..) | &AggregateKind::Tuple => {} - &AggregateKind::Adt(ref def, ..) => { - match self.tcx.layout_scalar_valid_range(def.did) { + &AggregateKind::Adt(adt_did, ..) => { + match self.tcx.layout_scalar_valid_range(adt_did) { (Bound::Unbounded, Bound::Unbounded) => {} _ => self.require_unsafe( UnsafetyViolationKind::General, @@ -243,7 +239,7 @@ impl<'a, 'tcx> Visitor<'tcx> for UnsafetyChecker<'a, 'tcx> { } } -impl<'a, 'tcx> UnsafetyChecker<'a, 'tcx> { +impl<'tcx> UnsafetyChecker<'_, 'tcx> { fn require_unsafe(&mut self, kind: UnsafetyViolationKind, details: UnsafetyViolationDetails) { // Violations can turn out to be `UnsafeFn` during analysis, but they should not start out as such. assert_ne!(kind, UnsafetyViolationKind::UnsafeFn); @@ -396,13 +392,7 @@ struct UnusedUnsafeVisitor<'a> { unsafe_blocks: &'a mut Vec<(hir::HirId, bool)>, } -impl<'a, 'tcx> intravisit::Visitor<'tcx> for UnusedUnsafeVisitor<'a> { - type Map = intravisit::ErasedMap<'tcx>; - - fn nested_visit_map(&mut self) -> intravisit::NestedVisitorMap<Self::Map> { - intravisit::NestedVisitorMap::None - } - +impl<'tcx> intravisit::Visitor<'tcx> for UnusedUnsafeVisitor<'_> { fn visit_block(&mut self, block: &'tcx hir::Block<'tcx>) { intravisit::walk_block(self, block); diff --git a/compiler/rustc_mir_transform/src/const_debuginfo.rs b/compiler/rustc_mir_transform/src/const_debuginfo.rs index b613634560f..839d94167fe 100644 --- a/compiler/rustc_mir_transform/src/const_debuginfo.rs +++ b/compiler/rustc_mir_transform/src/const_debuginfo.rs @@ -15,11 +15,11 @@ use rustc_index::{bit_set::BitSet, vec::IndexVec}; pub struct ConstDebugInfo; impl<'tcx> MirPass<'tcx> for ConstDebugInfo { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if !tcx.sess.opts.debugging_opts.unsound_mir_opts { - return; - } + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.opts.debugging_opts.unsound_mir_opts && sess.mir_opt_level() > 0 + } + fn run_pass(&self, _: TyCtxt<'tcx>, body: &mut Body<'tcx>) { trace!("running ConstDebugInfo on {:?}", body.source); for (local, constant) in find_optimization_oportunities(body) { @@ -89,7 +89,7 @@ fn find_optimization_oportunities<'tcx>(body: &Body<'tcx>) -> Vec<(Local, Consta eligable_locals } -impl<'tcx> Visitor<'tcx> for LocalUseVisitor { +impl Visitor<'_> for LocalUseVisitor { fn visit_local(&mut self, local: &Local, context: PlaceContext, location: Location) { if context.is_mutating_use() { self.local_mutating_uses[*local] = self.local_mutating_uses[*local].saturating_add(1); diff --git a/compiler/rustc_mir_transform/src/const_goto.rs b/compiler/rustc_mir_transform/src/const_goto.rs index d319fdcaa6b..905173b0457 100644 --- a/compiler/rustc_mir_transform/src/const_goto.rs +++ b/compiler/rustc_mir_transform/src/const_goto.rs @@ -27,10 +27,11 @@ use super::simplify::{simplify_cfg, simplify_locals}; pub struct ConstGoto; impl<'tcx> MirPass<'tcx> for ConstGoto { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() >= 4 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if tcx.sess.mir_opt_level() < 4 { - return; - } trace!("Running ConstGoto on {:?}", body.source); let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); let mut opt_finder = @@ -53,7 +54,7 @@ impl<'tcx> MirPass<'tcx> for ConstGoto { } } -impl<'a, 'tcx> Visitor<'tcx> for ConstGotoOptimizationFinder<'a, 'tcx> { +impl<'tcx> Visitor<'tcx> for ConstGotoOptimizationFinder<'_, 'tcx> { fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) { let _: Option<_> = try { let target = terminator.kind.as_goto()?; @@ -82,20 +83,7 @@ impl<'a, 'tcx> Visitor<'tcx> for ConstGotoOptimizationFinder<'a, 'tcx> { // Now find which value in the Switch matches the const value. let const_value = _const.literal.try_eval_bits(self.tcx, self.param_env, switch_ty)?; - let found_value_idx_option = targets - .iter() - .enumerate() - .find(|(_, (value, _))| const_value == *value) - .map(|(idx, _)| idx); - - let target_to_use_in_goto = - if let Some(found_value_idx) = found_value_idx_option { - targets.iter().nth(found_value_idx).unwrap().1 - } else { - // If we did not find the const value in values, it must be the otherwise case - targets.otherwise() - }; - + let target_to_use_in_goto = targets.target_for_value(const_value); self.optimizations.push(OptimizationToApply { bb_with_goto: location.block, target_to_use_in_goto, diff --git a/compiler/rustc_mir_transform/src/const_prop.rs b/compiler/rustc_mir_transform/src/const_prop.rs index 63c637af5c2..98de64cd97c 100644 --- a/compiler/rustc_mir_transform/src/const_prop.rs +++ b/compiler/rustc_mir_transform/src/const_prop.rs @@ -62,6 +62,13 @@ macro_rules! throw_machine_stop_str { pub struct ConstProp; impl<'tcx> MirPass<'tcx> for ConstProp { + fn is_enabled(&self, _sess: &rustc_session::Session) -> bool { + // FIXME(#70073): Unlike the other passes in "optimizations", this one emits errors, so it + // runs even when MIR optimizations are disabled. We should separate the lint out from the + // transform and move the lint as early in the pipeline as possible. + true + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // will be evaluated by miri and produce its errors there if body.source.promoted.is_some() { @@ -69,10 +76,8 @@ impl<'tcx> MirPass<'tcx> for ConstProp { } let def_id = body.source.def_id().expect_local(); - let hir_id = tcx.hir().local_def_id_to_hir_id(def_id); - - let is_fn_like = tcx.hir().get(hir_id).fn_kind().is_some(); - let is_assoc_const = tcx.def_kind(def_id.to_def_id()) == DefKind::AssocConst; + let is_fn_like = tcx.hir().get_by_def_id(def_id).fn_kind().is_some(); + let is_assoc_const = tcx.def_kind(def_id) == DefKind::AssocConst; // Only run const prop on functions, methods, closures and associated constants if !is_fn_like && !is_assoc_const { @@ -119,7 +124,7 @@ impl<'tcx> MirPass<'tcx> for ConstProp { .predicates_of(def_id.to_def_id()) .predicates .iter() - .filter_map(|(p, _)| if p.is_global(tcx) { Some(*p) } else { None }); + .filter_map(|(p, _)| if p.is_global() { Some(*p) } else { None }); if traits::impossible_predicates( tcx, traits::elaborate_predicates(tcx, predicates).map(|o| o.predicate).collect(), @@ -131,7 +136,6 @@ impl<'tcx> MirPass<'tcx> for ConstProp { trace!("ConstProp starting for {:?}", def_id); let dummy_body = &Body::new( - tcx, body.source, body.basic_blocks().clone(), body.source_scopes.clone(), @@ -164,7 +168,7 @@ struct ConstPropMachine<'mir, 'tcx> { can_const_prop: IndexVec<Local, ConstPropMode>, } -impl<'mir, 'tcx> ConstPropMachine<'mir, 'tcx> { +impl ConstPropMachine<'_, '_> { fn new( only_propagate_inside_block_locals: BitSet<Local>, can_const_prop: IndexVec<Local, ConstPropMode>, @@ -200,7 +204,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> _args: &[OpTy<'tcx>], _ret: Option<(&PlaceTy<'tcx>, BasicBlock)>, _unwind: StackPopUnwind, - ) -> InterpResult<'tcx, Option<&'mir Body<'tcx>>> { + ) -> InterpResult<'tcx, Option<(&'mir Body<'tcx>, ty::Instance<'tcx>)>> { Ok(None) } @@ -232,13 +236,6 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> throw_machine_stop_str!("pointer arithmetic or comparisons aren't supported in ConstProp") } - fn box_alloc( - _ecx: &mut InterpCx<'mir, 'tcx, Self>, - _dest: &PlaceTy<'tcx>, - ) -> InterpResult<'tcx> { - throw_machine_stop_str!("can't const prop heap allocations") - } - fn access_local( _ecx: &InterpCx<'mir, 'tcx, Self>, frame: &Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>, @@ -301,14 +298,14 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx> } #[inline(always)] - fn stack( + fn stack<'a>( ecx: &'a InterpCx<'mir, 'tcx, Self>, ) -> &'a [Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>] { &ecx.machine.stack } #[inline(always)] - fn stack_mut( + fn stack_mut<'a>( ecx: &'a mut InterpCx<'mir, 'tcx, Self>, ) -> &'a mut Vec<Frame<'mir, 'tcx, Self::PointerTag, Self::FrameExtra>> { &mut ecx.machine.stack @@ -329,7 +326,7 @@ struct ConstPropagator<'mir, 'tcx> { source_info: Option<SourceInfo>, } -impl<'mir, 'tcx> LayoutOfHelpers<'tcx> for ConstPropagator<'mir, 'tcx> { +impl<'tcx> LayoutOfHelpers<'tcx> for ConstPropagator<'_, 'tcx> { type LayoutOfResult = Result<TyAndLayout<'tcx>, LayoutError<'tcx>>; #[inline] @@ -338,21 +335,21 @@ impl<'mir, 'tcx> LayoutOfHelpers<'tcx> for ConstPropagator<'mir, 'tcx> { } } -impl<'mir, 'tcx> HasDataLayout for ConstPropagator<'mir, 'tcx> { +impl HasDataLayout for ConstPropagator<'_, '_> { #[inline] fn data_layout(&self) -> &TargetDataLayout { &self.tcx.data_layout } } -impl<'mir, 'tcx> ty::layout::HasTyCtxt<'tcx> for ConstPropagator<'mir, 'tcx> { +impl<'tcx> ty::layout::HasTyCtxt<'tcx> for ConstPropagator<'_, 'tcx> { #[inline] fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } } -impl<'mir, 'tcx> ty::layout::HasParamEnv<'tcx> for ConstPropagator<'mir, 'tcx> { +impl<'tcx> ty::layout::HasParamEnv<'tcx> for ConstPropagator<'_, 'tcx> { #[inline] fn param_env(&self) -> ty::ParamEnv<'tcx> { self.param_env @@ -406,7 +403,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { Instance::new(def_id, substs), dummy_body, ret.as_ref(), - StackPopCleanup::None { cleanup: false }, + StackPopCleanup::Root { cleanup: false }, ) .expect("failed to push initial stack frame"); @@ -475,7 +472,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { /// Returns the value, if any, of evaluating `c`. fn eval_constant(&mut self, c: &Constant<'tcx>, source_info: SourceInfo) -> Option<OpTy<'tcx>> { // FIXME we need to revisit this for #67176 - if c.definitely_needs_subst(self.tcx) { + if c.needs_subst() { return None; } @@ -490,14 +487,14 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { // Promoteds must lint and not error as the user didn't ask for them ConstKind::Unevaluated(ty::Unevaluated { def: _, - substs_: _, + substs: _, promoted: Some(_), }) => true, // Out of backwards compatibility we cannot report hard errors in unused // generic functions using associated constants of the generic parameters. - _ => c.literal.definitely_needs_subst(*tcx), + _ => c.literal.needs_subst(), }, - ConstantKind::Val(_, ty) => ty.definitely_needs_subst(*tcx), + ConstantKind::Val(_, ty) => ty.needs_subst(), }; if lint_only { // Out of backwards compatibility we cannot report hard errors in unused @@ -728,7 +725,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { } // FIXME we need to revisit this for #67176 - if rvalue.definitely_needs_subst(self.tcx) { + if rvalue.needs_subst() { return None; } @@ -745,62 +742,44 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { rvalue: &Rvalue<'tcx>, place: Place<'tcx>, ) -> Option<()> { - self.use_ecx(|this| { - match rvalue { - Rvalue::BinaryOp(op, box (left, right)) - | Rvalue::CheckedBinaryOp(op, box (left, right)) => { - let l = this.ecx.eval_operand(left, None); - let r = this.ecx.eval_operand(right, None); - - let const_arg = match (l, r) { - (Ok(ref x), Err(_)) | (Err(_), Ok(ref x)) => this.ecx.read_immediate(x)?, - (Err(e), Err(_)) => return Err(e), - (Ok(_), Ok(_)) => { - this.ecx.eval_rvalue_into_place(rvalue, place)?; - return Ok(()); - } - }; - - let arg_value = const_arg.to_scalar()?.to_bits(const_arg.layout.size)?; - let dest = this.ecx.eval_place(place)?; - - match op { - BinOp::BitAnd => { - if arg_value == 0 { - this.ecx.write_immediate(*const_arg, &dest)?; - } - } - BinOp::BitOr => { - if arg_value == const_arg.layout.size.truncate(u128::MAX) - || (const_arg.layout.ty.is_bool() && arg_value == 1) - { - this.ecx.write_immediate(*const_arg, &dest)?; - } - } - BinOp::Mul => { - if const_arg.layout.ty.is_integral() && arg_value == 0 { - if let Rvalue::CheckedBinaryOp(_, _) = rvalue { - let val = Immediate::ScalarPair( - const_arg.to_scalar()?.into(), - Scalar::from_bool(false).into(), - ); - this.ecx.write_immediate(val, &dest)?; - } else { - this.ecx.write_immediate(*const_arg, &dest)?; - } - } - } - _ => { - this.ecx.eval_rvalue_into_place(rvalue, place)?; + self.use_ecx(|this| match rvalue { + Rvalue::BinaryOp(op, box (left, right)) + | Rvalue::CheckedBinaryOp(op, box (left, right)) => { + let l = this.ecx.eval_operand(left, None); + let r = this.ecx.eval_operand(right, None); + + let const_arg = match (l, r) { + (Ok(ref x), Err(_)) | (Err(_), Ok(ref x)) => this.ecx.read_immediate(x)?, + (Err(e), Err(_)) => return Err(e), + (Ok(_), Ok(_)) => return this.ecx.eval_rvalue_into_place(rvalue, place), + }; + + let arg_value = const_arg.to_scalar()?.to_bits(const_arg.layout.size)?; + let dest = this.ecx.eval_place(place)?; + + match op { + BinOp::BitAnd if arg_value == 0 => this.ecx.write_immediate(*const_arg, &dest), + BinOp::BitOr + if arg_value == const_arg.layout.size.truncate(u128::MAX) + || (const_arg.layout.ty.is_bool() && arg_value == 1) => + { + this.ecx.write_immediate(*const_arg, &dest) + } + BinOp::Mul if const_arg.layout.ty.is_integral() && arg_value == 0 => { + if let Rvalue::CheckedBinaryOp(_, _) = rvalue { + let val = Immediate::ScalarPair( + const_arg.to_scalar()?.into(), + Scalar::from_bool(false).into(), + ); + this.ecx.write_immediate(val, &dest) + } else { + this.ecx.write_immediate(*const_arg, &dest) } } - } - _ => { - this.ecx.eval_rvalue_into_place(rvalue, place)?; + _ => this.ecx.eval_rvalue_into_place(rvalue, place), } } - - Ok(()) + _ => this.ecx.eval_rvalue_into_place(rvalue, place), }) } @@ -964,7 +943,7 @@ struct CanConstProp { impl CanConstProp { /// Returns true if `local` can be propagated - fn check( + fn check<'tcx>( tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, body: &Body<'tcx>, @@ -1012,7 +991,7 @@ impl CanConstProp { } } -impl<'tcx> Visitor<'tcx> for CanConstProp { +impl Visitor<'_> for CanConstProp { fn visit_local(&mut self, &local: &Local, context: PlaceContext, _: Location) { use rustc_middle::mir::visit::PlaceContext::*; match context { @@ -1022,6 +1001,7 @@ impl<'tcx> Visitor<'tcx> for CanConstProp { // These are just stores, where the storing is not propagatable, but there may be later // mutations of the same local via `Store` | MutatingUse(MutatingUseContext::Call) + | MutatingUse(MutatingUseContext::AsmOutput) // Actual store that can possibly even propagate a value | MutatingUse(MutatingUseContext::Store) => { if !self.found_assignment.insert(local) { @@ -1052,8 +1032,7 @@ impl<'tcx> Visitor<'tcx> for CanConstProp { // These could be propagated with a smarter analysis or just some careful thinking about // whether they'd be fine right now. - MutatingUse(MutatingUseContext::AsmOutput) - | MutatingUse(MutatingUseContext::Yield) + MutatingUse(MutatingUseContext::Yield) | MutatingUse(MutatingUseContext::Drop) | MutatingUse(MutatingUseContext::Retag) // These can't ever be propagated under any scheme, as we can't reason about indirect @@ -1071,7 +1050,7 @@ impl<'tcx> Visitor<'tcx> for CanConstProp { } } -impl<'mir, 'tcx> MutVisitor<'tcx> for ConstPropagator<'mir, 'tcx> { +impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } diff --git a/compiler/rustc_mir_transform/src/coverage/debug.rs b/compiler/rustc_mir_transform/src/coverage/debug.rs index 513a85b5913..ce8b187a744 100644 --- a/compiler/rustc_mir_transform/src/coverage/debug.rs +++ b/compiler/rustc_mir_transform/src/coverage/debug.rs @@ -111,6 +111,7 @@ use super::graph::{BasicCoverageBlock, BasicCoverageBlockData, CoverageGraph}; use super::spans::CoverageSpan; +use itertools::Itertools; use rustc_middle::mir::create_dump_file; use rustc_middle::mir::generic_graphviz::GraphvizWriter; use rustc_middle::mir::spanview::{self, SpanViewable}; @@ -148,7 +149,7 @@ impl DebugOptions { let mut counter_format = ExpressionFormat::default(); if let Ok(env_debug_options) = std::env::var(RUSTC_COVERAGE_DEBUG_OPTIONS) { - for setting_str in env_debug_options.replace(" ", "").replace("-", "_").split(',') { + for setting_str in env_debug_options.replace(' ', "").replace('-', "_").split(',') { let (option, value) = match setting_str.split_once('=') { None => (setting_str, None), Some((k, v)) => (k, Some(v)), @@ -629,7 +630,7 @@ impl UsedExpressions { } /// Generates the MIR pass `CoverageSpan`-specific spanview dump file. -pub(super) fn dump_coverage_spanview( +pub(super) fn dump_coverage_spanview<'tcx>( tcx: TyCtxt<'tcx>, mir_body: &mir::Body<'tcx>, basic_coverage_blocks: &CoverageGraph, @@ -651,7 +652,7 @@ pub(super) fn dump_coverage_spanview( } /// Converts the computed `BasicCoverageBlockData`s into `SpanViewable`s. -fn span_viewables( +fn span_viewables<'tcx>( tcx: TyCtxt<'tcx>, mir_body: &mir::Body<'tcx>, basic_coverage_blocks: &CoverageGraph, @@ -670,7 +671,7 @@ fn span_viewables( } /// Generates the MIR pass coverage-specific graphviz dump file. -pub(super) fn dump_coverage_graphviz( +pub(super) fn dump_coverage_graphviz<'tcx>( tcx: TyCtxt<'tcx>, mir_body: &mir::Body<'tcx>, pass_name: &str, @@ -739,7 +740,6 @@ pub(super) fn dump_coverage_graphviz( ) } }) - .collect::<Vec<_>>() .join("\n ") )); } @@ -750,7 +750,7 @@ pub(super) fn dump_coverage_graphviz( .expect("Unexpected error writing BasicCoverageBlock graphviz DOT file"); } -fn bcb_to_string_sections( +fn bcb_to_string_sections<'tcx>( tcx: TyCtxt<'tcx>, mir_body: &mir::Body<'tcx>, debug_counters: &DebugCounters, @@ -768,7 +768,6 @@ fn bcb_to_string_sections( .map(|expression| { format!("Intermediate {}", debug_counters.format_counter(expression)) }) - .collect::<Vec<_>>() .join("\n"), ); } @@ -783,7 +782,6 @@ fn bcb_to_string_sections( covspan.format(tcx, mir_body) ) }) - .collect::<Vec<_>>() .join("\n"), ); } @@ -793,7 +791,6 @@ fn bcb_to_string_sections( dependency_counters .iter() .map(|counter| debug_counters.format_counter(counter)) - .collect::<Vec<_>>() .join(" \n"), )); } @@ -817,7 +814,7 @@ fn bcb_to_string_sections( /// Returns a simple string representation of a `TerminatorKind` variant, independent of any /// values it might hold. -pub(super) fn term_type(kind: &TerminatorKind<'tcx>) -> &'static str { +pub(super) fn term_type(kind: &TerminatorKind<'_>) -> &'static str { match kind { TerminatorKind::Goto { .. } => "Goto", TerminatorKind::SwitchInt { .. } => "SwitchInt", diff --git a/compiler/rustc_mir_transform/src/coverage/graph.rs b/compiler/rustc_mir_transform/src/coverage/graph.rs index d78ad6ce97f..57862b6628d 100644 --- a/compiler/rustc_mir_transform/src/coverage/graph.rs +++ b/compiler/rustc_mir_transform/src/coverage/graph.rs @@ -1,5 +1,6 @@ use super::Error; +use itertools::Itertools; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::graph::dominators::{self, Dominators}; use rustc_data_structures::graph::{self, GraphSuccessors, WithNumNodes, WithStartNode}; @@ -27,7 +28,7 @@ pub(super) struct CoverageGraph { } impl CoverageGraph { - pub fn from_mir(mir_body: &mir::Body<'tcx>) -> Self { + pub fn from_mir(mir_body: &mir::Body<'_>) -> Self { let (bcbs, bb_to_bcb) = Self::compute_basic_coverage_blocks(mir_body); // Pre-transform MIR `BasicBlock` successors and predecessors into the BasicCoverageBlock @@ -74,7 +75,7 @@ impl CoverageGraph { } fn compute_basic_coverage_blocks( - mir_body: &mir::Body<'tcx>, + mir_body: &mir::Body<'_>, ) -> ( IndexVec<BasicCoverageBlock, BasicCoverageBlockData>, IndexVec<BasicBlock, Option<BasicCoverageBlock>>, @@ -267,7 +268,7 @@ impl graph::WithSuccessors for CoverageGraph { } } -impl graph::GraphPredecessors<'graph> for CoverageGraph { +impl<'graph> graph::GraphPredecessors<'graph> for CoverageGraph { type Item = BasicCoverageBlock; type Iter = std::iter::Copied<std::slice::Iter<'graph, BasicCoverageBlock>>; } @@ -418,18 +419,11 @@ impl BasicCoverageBlockData { pub fn take_edge_counters( &mut self, ) -> Option<impl Iterator<Item = (BasicCoverageBlock, CoverageKind)>> { - self.edge_from_bcbs.take().map_or(None, |m| Some(m.into_iter())) + self.edge_from_bcbs.take().map(|m| m.into_iter()) } pub fn id(&self) -> String { - format!( - "@{}", - self.basic_blocks - .iter() - .map(|bb| bb.index().to_string()) - .collect::<Vec<_>>() - .join(ID_SEPARATOR) - ) + format!("@{}", self.basic_blocks.iter().map(|bb| bb.index().to_string()).join(ID_SEPARATOR)) } } diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs index 6807d02519e..82455654a88 100644 --- a/compiler/rustc_mir_transform/src/coverage/mod.rs +++ b/compiler/rustc_mir_transform/src/coverage/mod.rs @@ -49,6 +49,10 @@ impl Error { pub struct InstrumentCoverage; impl<'tcx> MirPass<'tcx> for InstrumentCoverage { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.instrument_coverage() + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, mir_body: &mut mir::Body<'tcx>) { let mir_source = mir_body.source; @@ -62,8 +66,8 @@ impl<'tcx> MirPass<'tcx> for InstrumentCoverage { return; } - let hir_id = tcx.hir().local_def_id_to_hir_id(mir_source.def_id().expect_local()); - let is_fn_like = tcx.hir().get(hir_id).fn_kind().is_some(); + let is_fn_like = + tcx.hir().get_by_def_id(mir_source.def_id().expect_local()).fn_kind().is_some(); // Only instrument functions, methods, and closures (not constants since they are evaluated // at compile time by Miri). @@ -439,7 +443,7 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> { } fn inject_edge_counter_basic_block( - mir_body: &mut mir::Body<'tcx>, + mir_body: &mut mir::Body<'_>, from_bb: BasicBlock, to_bb: BasicBlock, ) -> BasicBlock { @@ -462,7 +466,7 @@ fn inject_edge_counter_basic_block( } fn inject_statement( - mir_body: &mut mir::Body<'tcx>, + mir_body: &mut mir::Body<'_>, counter_kind: CoverageKind, bb: BasicBlock, some_code_region: Option<CodeRegion>, @@ -484,7 +488,7 @@ fn inject_statement( } // Non-code expressions are injected into the coverage map, without generating executable code. -fn inject_intermediate_expression(mir_body: &mut mir::Body<'tcx>, expression: CoverageKind) { +fn inject_intermediate_expression(mir_body: &mut mir::Body<'_>, expression: CoverageKind) { debug_assert!(matches!(expression, CoverageKind::Expression { .. })); debug!(" injecting non-code expression {:?}", expression); let inject_in_bb = mir::START_BLOCK; diff --git a/compiler/rustc_mir_transform/src/coverage/query.rs b/compiler/rustc_mir_transform/src/coverage/query.rs index 760f16eae6b..46de6d939a1 100644 --- a/compiler/rustc_mir_transform/src/coverage/query.rs +++ b/compiler/rustc_mir_transform/src/coverage/query.rs @@ -9,7 +9,6 @@ use rustc_span::def_id::DefId; /// A `query` provider for retrieving coverage information injected into MIR. pub(crate) fn provide(providers: &mut Providers) { providers.coverageinfo = |tcx, def_id| coverageinfo(tcx, def_id); - providers.covered_file_name = |tcx, def_id| covered_file_name(tcx, def_id); providers.covered_code_regions = |tcx, def_id| covered_code_regions(tcx, def_id); } @@ -137,25 +136,6 @@ fn coverageinfo<'tcx>(tcx: TyCtxt<'tcx>, instance_def: ty::InstanceDef<'tcx>) -> coverage_visitor.info } -fn covered_file_name<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> Option<Symbol> { - if tcx.is_mir_available(def_id) { - let body = mir_body(tcx, def_id); - for bb_data in body.basic_blocks().iter() { - for statement in bb_data.statements.iter() { - if let StatementKind::Coverage(box ref coverage) = statement.kind { - if let Some(code_region) = coverage.code_region.as_ref() { - if is_inlined(body, statement) { - continue; - } - return Some(code_region.file_name); - } - } - } - } - } - return None; -} - fn covered_code_regions<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> Vec<&'tcx CodeRegion> { let body = mir_body(tcx, def_id); body.basic_blocks() diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs index d13fa0729cd..d1cb2826ded 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans.rs @@ -1,6 +1,7 @@ use super::debug::term_type; use super::graph::{BasicCoverageBlock, BasicCoverageBlockData, CoverageGraph, START_BCB}; +use itertools::Itertools; use rustc_data_structures::graph::WithNumNodes; use rustc_middle::mir::spanview::source_range_no_file; use rustc_middle::mir::{ @@ -21,13 +22,13 @@ pub(super) enum CoverageStatement { } impl CoverageStatement { - pub fn format(&self, tcx: TyCtxt<'tcx>, mir_body: &'a mir::Body<'tcx>) -> String { + pub fn format<'tcx>(&self, tcx: TyCtxt<'tcx>, mir_body: &mir::Body<'tcx>) -> String { match *self { Self::Statement(bb, span, stmt_index) => { let stmt = &mir_body[bb].statements[stmt_index]; format!( "{}: @{}[{}]: {:?}", - source_range_no_file(tcx, &span), + source_range_no_file(tcx, span), bb.index(), stmt_index, stmt @@ -37,7 +38,7 @@ impl CoverageStatement { let term = mir_body[bb].terminator(); format!( "{}: @{}.{}: {:?}", - source_range_no_file(tcx, &span), + source_range_no_file(tcx, span), bb.index(), term_type(&term.kind), term.kind @@ -86,7 +87,7 @@ impl CoverageSpan { } pub fn for_statement( - statement: &Statement<'tcx>, + statement: &Statement<'_>, span: Span, expn_span: Span, bcb: BasicCoverageBlock, @@ -151,29 +152,25 @@ impl CoverageSpan { self.bcb == other.bcb } - pub fn format(&self, tcx: TyCtxt<'tcx>, mir_body: &'a mir::Body<'tcx>) -> String { + pub fn format<'tcx>(&self, tcx: TyCtxt<'tcx>, mir_body: &mir::Body<'tcx>) -> String { format!( "{}\n {}", - source_range_no_file(tcx, &self.span), - self.format_coverage_statements(tcx, mir_body).replace("\n", "\n "), + source_range_no_file(tcx, self.span), + self.format_coverage_statements(tcx, mir_body).replace('\n', "\n "), ) } - pub fn format_coverage_statements( + pub fn format_coverage_statements<'tcx>( &self, tcx: TyCtxt<'tcx>, - mir_body: &'a mir::Body<'tcx>, + mir_body: &mir::Body<'tcx>, ) -> String { let mut sorted_coverage_statements = self.coverage_statements.clone(); sorted_coverage_statements.sort_unstable_by_key(|covstmt| match *covstmt { CoverageStatement::Statement(bb, _, index) => (bb, index), CoverageStatement::Terminator(bb, _) => (bb, usize::MAX), }); - sorted_coverage_statements - .iter() - .map(|covstmt| covstmt.format(tcx, mir_body)) - .collect::<Vec<_>>() - .join("\n") + sorted_coverage_statements.iter().map(|covstmt| covstmt.format(tcx, mir_body)).join("\n") } /// If the span is part of a macro, returns the macro name symbol. @@ -329,9 +326,7 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { fn mir_to_initial_sorted_coverage_spans(&self) -> Vec<CoverageSpan> { let mut initial_spans = Vec::<CoverageSpan>::with_capacity(self.mir_body.num_nodes() * 2); for (bcb, bcb_data) in self.basic_coverage_blocks.iter_enumerated() { - for coverage_span in self.bcb_to_initial_coverage_spans(bcb, bcb_data) { - initial_spans.push(coverage_span); - } + initial_spans.extend(self.bcb_to_initial_coverage_spans(bcb, bcb_data)); } if initial_spans.is_empty() { @@ -803,7 +798,7 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { /// If the MIR `Statement` has a span contributive to computing coverage spans, /// return it; otherwise return `None`. -pub(super) fn filtered_statement_span(statement: &'a Statement<'tcx>) -> Option<Span> { +pub(super) fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span> { match statement.kind { // These statements have spans that are often outside the scope of the executed source code // for their parent `BasicBlock`. @@ -837,7 +832,6 @@ pub(super) fn filtered_statement_span(statement: &'a Statement<'tcx>) -> Option< | StatementKind::CopyNonOverlapping(..) | StatementKind::Assign(_) | StatementKind::SetDiscriminant { .. } - | StatementKind::LlvmInlineAsm(_) | StatementKind::Retag(_, _) | StatementKind::AscribeUserType(_, _) => { Some(statement.source_info.span) @@ -847,7 +841,7 @@ pub(super) fn filtered_statement_span(statement: &'a Statement<'tcx>) -> Option< /// If the MIR `Terminator` has a span contributive to computing coverage spans, /// return it; otherwise return `None`. -pub(super) fn filtered_terminator_span(terminator: &'a Terminator<'tcx>) -> Option<Span> { +pub(super) fn filtered_terminator_span(terminator: &Terminator<'_>) -> Option<Span> { match terminator.kind { // These terminators have spans that don't positively contribute to computing a reasonable // span of actually executed source code. (For example, SwitchInt terminators extracted from diff --git a/compiler/rustc_mir_transform/src/coverage/tests.rs b/compiler/rustc_mir_transform/src/coverage/tests.rs index 14dd0a8b924..62ea2538ff0 100644 --- a/compiler/rustc_mir_transform/src/coverage/tests.rs +++ b/compiler/rustc_mir_transform/src/coverage/tests.rs @@ -31,6 +31,7 @@ use super::spans; use coverage_test_macros::let_bcb; +use itertools::Itertools; use rustc_data_structures::graph::WithNumNodes; use rustc_data_structures::graph::WithSuccessors; use rustc_index::vec::{Idx, IndexVec}; @@ -180,7 +181,7 @@ impl<'tcx> MockBlocks<'tcx> { } } -fn debug_basic_blocks(mir_body: &Body<'tcx>) -> String { +fn debug_basic_blocks<'tcx>(mir_body: &Body<'tcx>) -> String { format!( "{:?}", mir_body @@ -232,11 +233,9 @@ fn print_mir_graphviz(name: &str, mir_body: &Body<'_>) { mir_body .successors(bb) .map(|successor| { format!(" {:?} -> {:?};", bb, successor) }) - .collect::<Vec<_>>() .join("\n") ) }) - .collect::<Vec<_>>() .join("\n") ); } @@ -262,18 +261,16 @@ fn print_coverage_graphviz( basic_coverage_blocks .successors(bcb) .map(|successor| { format!(" {:?} -> {:?};", bcb, successor) }) - .collect::<Vec<_>>() .join("\n") ) }) - .collect::<Vec<_>>() .join("\n") ); } } /// Create a mock `Body` with a simple flow. -fn goto_switchint() -> Body<'a> { +fn goto_switchint<'a>() -> Body<'a> { let mut blocks = MockBlocks::new(); let start = blocks.call(None); let goto = blocks.goto(Some(start)); @@ -363,7 +360,7 @@ fn test_covgraph_goto_switchint() { } /// Create a mock `Body` with a loop. -fn switchint_then_loop_else_return() -> Body<'a> { +fn switchint_then_loop_else_return<'a>() -> Body<'a> { let mut blocks = MockBlocks::new(); let start = blocks.call(None); let switchint = blocks.switchint(Some(start)); @@ -449,7 +446,7 @@ fn test_covgraph_switchint_then_loop_else_return() { } /// Create a mock `Body` with nested loops. -fn switchint_loop_then_inner_loop_else_break() -> Body<'a> { +fn switchint_loop_then_inner_loop_else_break<'a>() -> Body<'a> { let mut blocks = MockBlocks::new(); let start = blocks.call(None); let switchint = blocks.switchint(Some(start)); diff --git a/compiler/rustc_mir_transform/src/deduplicate_blocks.rs b/compiler/rustc_mir_transform/src/deduplicate_blocks.rs index 8d2413433a9..d1977ed49fe 100644 --- a/compiler/rustc_mir_transform/src/deduplicate_blocks.rs +++ b/compiler/rustc_mir_transform/src/deduplicate_blocks.rs @@ -15,10 +15,11 @@ use super::simplify::simplify_cfg; pub struct DeduplicateBlocks; impl<'tcx> MirPass<'tcx> for DeduplicateBlocks { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() >= 4 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if tcx.sess.mir_opt_level() < 4 { - return; - } debug!("Running DeduplicateBlocks on `{:?}`", body.source); let duplicates = find_duplicates(body); let has_opts_to_apply = !duplicates.is_empty(); @@ -53,7 +54,7 @@ impl<'tcx> MutVisitor<'tcx> for OptApplier<'tcx> { } } -fn find_duplicates<'a, 'tcx>(body: &'a Body<'tcx>) -> FxHashMap<BasicBlock, BasicBlock> { +fn find_duplicates(body: &Body<'_>) -> FxHashMap<BasicBlock, BasicBlock> { let mut duplicates = FxHashMap::default(); let bbs_to_go_through = @@ -101,7 +102,7 @@ struct BasicBlockHashable<'tcx, 'a> { basic_block_data: &'a BasicBlockData<'tcx>, } -impl<'tcx, 'a> Hash for BasicBlockHashable<'tcx, 'a> { +impl Hash for BasicBlockHashable<'_, '_> { fn hash<H: Hasher>(&self, state: &mut H) { hash_statements(state, self.basic_block_data.statements.iter()); // Note that since we only hash the kind, we lose span information if we deduplicate the blocks @@ -109,9 +110,9 @@ impl<'tcx, 'a> Hash for BasicBlockHashable<'tcx, 'a> { } } -impl<'tcx, 'a> Eq for BasicBlockHashable<'tcx, 'a> {} +impl Eq for BasicBlockHashable<'_, '_> {} -impl<'tcx, 'a> PartialEq for BasicBlockHashable<'tcx, 'a> { +impl PartialEq for BasicBlockHashable<'_, '_> { fn eq(&self, other: &Self) -> bool { self.basic_block_data.statements.len() == other.basic_block_data.statements.len() && &self.basic_block_data.terminator().kind == &other.basic_block_data.terminator().kind @@ -131,7 +132,7 @@ fn hash_statements<'a, 'tcx, H: Hasher>( } } -fn statement_hash<'tcx, H: Hasher>(hasher: &mut H, stmt: &StatementKind<'tcx>) { +fn statement_hash<H: Hasher>(hasher: &mut H, stmt: &StatementKind<'_>) { match stmt { StatementKind::Assign(box (place, rvalue)) => { place.hash(hasher); @@ -141,14 +142,14 @@ fn statement_hash<'tcx, H: Hasher>(hasher: &mut H, stmt: &StatementKind<'tcx>) { }; } -fn rvalue_hash<H: Hasher>(hasher: &mut H, rvalue: &Rvalue<'tcx>) { +fn rvalue_hash<H: Hasher>(hasher: &mut H, rvalue: &Rvalue<'_>) { match rvalue { Rvalue::Use(op) => operand_hash(hasher, op), x => x.hash(hasher), }; } -fn operand_hash<H: Hasher>(hasher: &mut H, operand: &Operand<'tcx>) { +fn operand_hash<H: Hasher>(hasher: &mut H, operand: &Operand<'_>) { match operand { Operand::Constant(box Constant { user_ty: _, literal, span: _ }) => literal.hash(hasher), x => x.hash(hasher), @@ -167,7 +168,7 @@ fn statement_eq<'tcx>(lhs: &StatementKind<'tcx>, rhs: &StatementKind<'tcx>) -> b res } -fn rvalue_eq(lhs: &Rvalue<'tcx>, rhs: &Rvalue<'tcx>) -> bool { +fn rvalue_eq<'tcx>(lhs: &Rvalue<'tcx>, rhs: &Rvalue<'tcx>) -> bool { let res = match (lhs, rhs) { (Rvalue::Use(op1), Rvalue::Use(op2)) => operand_eq(op1, op2), (x, y) => x == y, @@ -176,7 +177,7 @@ fn rvalue_eq(lhs: &Rvalue<'tcx>, rhs: &Rvalue<'tcx>) -> bool { res } -fn operand_eq(lhs: &Operand<'tcx>, rhs: &Operand<'tcx>) -> bool { +fn operand_eq<'tcx>(lhs: &Operand<'tcx>, rhs: &Operand<'tcx>) -> bool { let res = match (lhs, rhs) { ( Operand::Constant(box Constant { user_ty: _, literal, span: _ }), diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index 790d9243fba..d469be74641 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -124,18 +124,15 @@ const MAX_BLOCKS: usize = 250; pub struct DestinationPropagation; impl<'tcx> MirPass<'tcx> for DestinationPropagation { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - // FIXME(#79191, #82678) - if !tcx.sess.opts.debugging_opts.unsound_mir_opts { - return; - } - + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + // FIXME(#79191, #82678): This is unsound. + // // Only run at mir-opt-level=3 or higher for now (we don't fix up debuginfo and remove // storage statements at the moment). - if tcx.sess.mir_opt_level() < 3 { - return; - } + sess.opts.debugging_opts.unsound_mir_opts && sess.mir_opt_level() >= 3 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let def_id = body.source.def_id(); let candidates = find_candidates(tcx, body); @@ -244,7 +241,7 @@ struct Replacements<'tcx> { kill: BitSet<Local>, } -impl Replacements<'tcx> { +impl<'tcx> Replacements<'tcx> { fn new(locals: usize) -> Self { Self { map: IndexVec::from_elem_n(None, locals), kill: BitSet::new_empty(locals) } } @@ -301,7 +298,7 @@ struct Replacer<'tcx> { } impl<'tcx> MutVisitor<'tcx> for Replacer<'tcx> { - fn tcx<'a>(&'a self) -> TyCtxt<'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } @@ -316,28 +313,6 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'tcx> { } } - fn process_projection_elem( - &mut self, - elem: PlaceElem<'tcx>, - _: Location, - ) -> Option<PlaceElem<'tcx>> { - match elem { - PlaceElem::Index(local) => { - if let Some(replacement) = self.replacements.for_src(local) { - bug!( - "cannot replace {:?} with {:?} in index projection {:?}", - local, - replacement, - elem, - ); - } else { - None - } - } - _ => None, - } - } - fn visit_place(&mut self, place: &mut Place<'tcx>, context: PlaceContext, location: Location) { if let Some(replacement) = self.replacements.for_src(place.local) { // Rebase `place`s projections onto `replacement`'s. @@ -397,7 +372,7 @@ struct Conflicts<'a> { unified_locals: InPlaceUnificationTable<UnifyLocal>, } -impl Conflicts<'a> { +impl<'a> Conflicts<'a> { fn build<'tcx>( tcx: TyCtxt<'tcx>, body: &'_ Body<'tcx>, @@ -559,25 +534,6 @@ impl Conflicts<'a> { // eliminate the resulting self-assignments automatically. StatementKind::Assign(_) => {} - StatementKind::LlvmInlineAsm(asm) => { - // Inputs and outputs must not overlap. - for (_, input) in &*asm.inputs { - if let Some(in_place) = input.place() { - if !in_place.is_indirect() { - for out_place in &*asm.outputs { - if !out_place.is_indirect() && !in_place.is_indirect() { - self.record_local_conflict( - in_place.local, - out_place.local, - "aliasing llvm_asm! operands", - ); - } - } - } - } - } - } - StatementKind::SetDiscriminant { .. } | StatementKind::StorageLive(..) | StatementKind::StorageDead(..) @@ -646,6 +602,7 @@ impl Conflicts<'a> { options: _, line_spans: _, destination: _, + cleanup: _, } => { // The intended semantics here aren't documented, we just assume that nothing that // could be written to by the assembly may overlap with any other operands. @@ -844,10 +801,7 @@ struct CandidateAssignment<'tcx> { /// comment) and also throw out assignments that involve a local that has its address taken or is /// otherwise ineligible (eg. locals used as array indices are ignored because we cannot propagate /// arbitrary places into array indices). -fn find_candidates<'a, 'tcx>( - tcx: TyCtxt<'tcx>, - body: &'a Body<'tcx>, -) -> Vec<CandidateAssignment<'tcx>> { +fn find_candidates<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> Vec<CandidateAssignment<'tcx>> { let mut visitor = FindAssignments { tcx, body, @@ -867,7 +821,7 @@ struct FindAssignments<'a, 'tcx> { locals_used_as_array_index: BitSet<Local>, } -impl<'a, 'tcx> Visitor<'tcx> for FindAssignments<'a, 'tcx> { +impl<'tcx> Visitor<'tcx> for FindAssignments<'_, 'tcx> { fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { if let StatementKind::Assign(box ( dest, diff --git a/compiler/rustc_mir_transform/src/early_otherwise_branch.rs b/compiler/rustc_mir_transform/src/early_otherwise_branch.rs index f191911a6c7..9a6b6532ce8 100644 --- a/compiler/rustc_mir_transform/src/early_otherwise_branch.rs +++ b/compiler/rustc_mir_transform/src/early_otherwise_branch.rs @@ -1,12 +1,12 @@ use rustc_middle::mir::patch::MirPatch; use rustc_middle::mir::*; -use rustc_middle::ty::{Ty, TyCtxt}; +use rustc_middle::ty::{self, Ty, TyCtxt}; use std::fmt::Debug; use super::simplify::simplify_cfg; /// This pass optimizes something like -/// ```text +/// ```ignore (syntax-highlighting-only) /// let x: Option<()>; /// let y: Option<()>; /// match (x,y) { @@ -15,146 +15,201 @@ use super::simplify::simplify_cfg; /// } /// ``` /// into something like -/// ```text +/// ```ignore (syntax-highlighting-only) /// let x: Option<()>; /// let y: Option<()>; -/// let discriminant_x = // get discriminant of x -/// let discriminant_y = // get discriminant of y -/// if discriminant_x != discriminant_y || discriminant_x == None {1} else {0} +/// let discriminant_x = std::mem::discriminant(x); +/// let discriminant_y = std::mem::discriminant(y); +/// if discriminant_x == discriminant_y { +/// match x { +/// Some(_) => 0, +/// _ => 1, // <---- +/// } // | Actually the same bb +/// } else { // | +/// 1 // <-------------- +/// } +/// ``` +/// +/// Specifically, it looks for instances of control flow like this: +/// ```text +/// +/// ================= +/// | BB1 | +/// |---------------| ============================ +/// | ... | /------> | BBC | +/// |---------------| | |--------------------------| +/// | switchInt(Q) | | | _cl = discriminant(P) | +/// | c | --------/ |--------------------------| +/// | d | -------\ | switchInt(_cl) | +/// | ... | | | c | ---> BBC.2 +/// | otherwise | --\ | /--- | otherwise | +/// ================= | | | ============================ +/// | | | +/// ================= | | | +/// | BBU | <-| | | ============================ +/// |---------------| | \-------> | BBD | +/// |---------------| | | |--------------------------| +/// | unreachable | | | | _dl = discriminant(P) | +/// ================= | | |--------------------------| +/// | | | switchInt(_dl) | +/// ================= | | | d | ---> BBD.2 +/// | BB9 | <--------------- | otherwise | +/// |---------------| ============================ +/// | ... | +/// ================= /// ``` +/// Where the `otherwise` branch on `BB1` is permitted to either go to `BBU` or to `BB9`. In the +/// code: +/// - `BB1` is `parent` and `BBC, BBD` are children +/// - `P` is `child_place` +/// - `child_ty` is the type of `_cl`. +/// - `Q` is `parent_op`. +/// - `parent_ty` is the type of `Q`. +/// - `BB9` is `destination` +/// All this is then transformed into: +/// ```text +/// +/// ======================= +/// | BB1 | +/// |---------------------| ============================ +/// | ... | /------> | BBEq | +/// | _s = discriminant(P)| | |--------------------------| +/// | _t = Ne(Q, _s) | | |--------------------------| +/// |---------------------| | | switchInt(Q) | +/// | switchInt(_t) | | | c | ---> BBC.2 +/// | false | --------/ | d | ---> BBD.2 +/// | otherwise | ---------------- | otherwise | +/// ======================= | ============================ +/// | +/// ================= | +/// | BB9 | <-----------/ +/// |---------------| +/// | ... | +/// ================= +/// ``` +/// +/// This is only correct for some `P`, since `P` is now computed outside the original `switchInt`. +/// The filter on which `P` are allowed (together with discussion of its correctness) is found in +/// `may_hoist`. pub struct EarlyOtherwiseBranch; impl<'tcx> MirPass<'tcx> for EarlyOtherwiseBranch { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - // FIXME(#78496) - if !tcx.sess.opts.debugging_opts.unsound_mir_opts { - return; - } + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() >= 2 + } - if tcx.sess.mir_opt_level() < 3 { - return; - } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { trace!("running EarlyOtherwiseBranch on {:?}", body.source); - // we are only interested in this bb if the terminator is a switchInt - let bbs_with_switch = - body.basic_blocks().iter_enumerated().filter(|(_, bb)| is_switch(bb.terminator())); - let opts_to_apply: Vec<OptimizationToApply<'tcx>> = bbs_with_switch - .flat_map(|(bb_idx, bb)| { - let switch = bb.terminator(); - let helper = Helper { body, tcx }; - let infos = helper.go(bb, switch)?; - Some(OptimizationToApply { infos, basic_block_first_switch: bb_idx }) - }) - .collect(); + let mut should_cleanup = false; - let should_cleanup = !opts_to_apply.is_empty(); + // Also consider newly generated bbs in the same pass + for i in 0..body.basic_blocks().len() { + let bbs = body.basic_blocks(); + let parent = BasicBlock::from_usize(i); + let Some(opt_data) = evaluate_candidate(tcx, body, parent) else { + continue + }; - for opt_to_apply in opts_to_apply { - if !tcx.consider_optimizing(|| format!("EarlyOtherwiseBranch {:?}", &opt_to_apply)) { + if !tcx.consider_optimizing(|| format!("EarlyOtherwiseBranch {:?}", &opt_data)) { break; } - trace!("SUCCESS: found optimization possibility to apply: {:?}", &opt_to_apply); + trace!("SUCCESS: found optimization possibility to apply: {:?}", &opt_data); - let statements_before = - body.basic_blocks()[opt_to_apply.basic_block_first_switch].statements.len(); - let end_of_block_location = Location { - block: opt_to_apply.basic_block_first_switch, - statement_index: statements_before, + should_cleanup = true; + + let TerminatorKind::SwitchInt { + discr: parent_op, + switch_ty: parent_ty, + targets: parent_targets + } = &bbs[parent].terminator().kind else { + unreachable!() + }; + // Always correct since we can only switch on `Copy` types + let parent_op = match parent_op { + Operand::Move(x) => Operand::Copy(*x), + Operand::Copy(x) => Operand::Copy(*x), + Operand::Constant(x) => Operand::Constant(x.clone()), }; + let statements_before = bbs[parent].statements.len(); + let parent_end = Location { block: parent, statement_index: statements_before }; let mut patch = MirPatch::new(body); - // create temp to store second discriminant in - let discr_type = opt_to_apply.infos[0].second_switch_info.discr_ty; - let discr_span = opt_to_apply.infos[0].second_switch_info.discr_source_info.span; - let second_discriminant_temp = patch.new_temp(discr_type, discr_span); + // create temp to store second discriminant in, `_s` in example above + let second_discriminant_temp = + patch.new_temp(opt_data.child_ty, opt_data.child_source.span); - patch.add_statement( - end_of_block_location, - StatementKind::StorageLive(second_discriminant_temp), - ); + patch.add_statement(parent_end, StatementKind::StorageLive(second_discriminant_temp)); // create assignment of discriminant - let place_of_adt_to_get_discriminant_of = - opt_to_apply.infos[0].second_switch_info.place_of_adt_discr_read; patch.add_assign( - end_of_block_location, + parent_end, Place::from(second_discriminant_temp), - Rvalue::Discriminant(place_of_adt_to_get_discriminant_of), + Rvalue::Discriminant(opt_data.child_place), ); - // create temp to store NotEqual comparison between the two discriminants - let not_equal = BinOp::Ne; - let not_equal_res_type = not_equal.ty(tcx, discr_type, discr_type); - let not_equal_temp = patch.new_temp(not_equal_res_type, discr_span); - patch.add_statement(end_of_block_location, StatementKind::StorageLive(not_equal_temp)); - - // create NotEqual comparison between the two discriminants - let first_descriminant_place = - opt_to_apply.infos[0].first_switch_info.discr_used_in_switch; - let not_equal_rvalue = Rvalue::BinaryOp( - not_equal, - Box::new(( - Operand::Copy(Place::from(second_discriminant_temp)), - Operand::Copy(first_descriminant_place), - )), + // create temp to store inequality comparison between the two discriminants, `_t` in + // example above + let nequal = BinOp::Ne; + let comp_res_type = nequal.ty(tcx, parent_ty, opt_data.child_ty); + let comp_temp = patch.new_temp(comp_res_type, opt_data.child_source.span); + patch.add_statement(parent_end, StatementKind::StorageLive(comp_temp)); + + // create inequality comparison between the two discriminants + let comp_rvalue = Rvalue::BinaryOp( + nequal, + Box::new((parent_op.clone(), Operand::Move(Place::from(second_discriminant_temp)))), ); patch.add_statement( - end_of_block_location, - StatementKind::Assign(Box::new((Place::from(not_equal_temp), not_equal_rvalue))), - ); - - let new_targets = opt_to_apply - .infos - .iter() - .flat_map(|x| x.second_switch_info.targets_with_values.iter()) - .cloned(); - - let targets = SwitchTargets::new( - new_targets, - opt_to_apply.infos[0].first_switch_info.otherwise_bb, + parent_end, + StatementKind::Assign(Box::new((Place::from(comp_temp), comp_rvalue))), ); - // new block that jumps to the correct discriminant case. This block is switched to if the discriminants are equal - let new_switch_data = BasicBlockData::new(Some(Terminator { - source_info: opt_to_apply.infos[0].second_switch_info.discr_source_info, + let eq_new_targets = parent_targets.iter().map(|(value, child)| { + let TerminatorKind::SwitchInt{ targets, .. } = &bbs[child].terminator().kind else { + unreachable!() + }; + (value, targets.target_for_value(value)) + }); + let eq_targets = SwitchTargets::new(eq_new_targets, opt_data.destination); + + // Create `bbEq` in example above + let eq_switch = BasicBlockData::new(Some(Terminator { + source_info: bbs[parent].terminator().source_info, kind: TerminatorKind::SwitchInt { - // the first and second discriminants are equal, so just pick one - discr: Operand::Copy(first_descriminant_place), - switch_ty: discr_type, - targets, + // switch on the first discriminant, so we can mark the second one as dead + discr: parent_op, + switch_ty: opt_data.child_ty, + targets: eq_targets, }, })); - let new_switch_bb = patch.new_block(new_switch_data); + let eq_bb = patch.new_block(eq_switch); - // switch on the NotEqual. If true, then jump to the `otherwise` case. - // If false, then jump to a basic block that then jumps to the correct disciminant case - let true_case = opt_to_apply.infos[0].first_switch_info.otherwise_bb; - let false_case = new_switch_bb; + // Jump to it on the basis of the inequality comparison + let true_case = opt_data.destination; + let false_case = eq_bb; patch.patch_terminator( - opt_to_apply.basic_block_first_switch, + parent, TerminatorKind::if_( tcx, - Operand::Move(Place::from(not_equal_temp)), + Operand::Move(Place::from(comp_temp)), true_case, false_case, ), ); // generate StorageDead for the second_discriminant_temp not in use anymore - patch.add_statement( - end_of_block_location, - StatementKind::StorageDead(second_discriminant_temp), - ); + patch.add_statement(parent_end, StatementKind::StorageDead(second_discriminant_temp)); - // Generate a StorageDead for not_equal_temp in each of the targets, since we moved it into the switch + // Generate a StorageDead for comp_temp in each of the targets, since we moved it into + // the switch for bb in [false_case, true_case].iter() { patch.add_statement( Location { block: *bb, statement_index: 0 }, - StatementKind::StorageDead(not_equal_temp), + StatementKind::StorageDead(comp_temp), ); } @@ -169,201 +224,177 @@ impl<'tcx> MirPass<'tcx> for EarlyOtherwiseBranch { } } -fn is_switch<'tcx>(terminator: &Terminator<'tcx>) -> bool { - matches!(terminator.kind, TerminatorKind::SwitchInt { .. }) -} - -struct Helper<'a, 'tcx> { - body: &'a Body<'tcx>, - tcx: TyCtxt<'tcx>, -} - -#[derive(Debug, Clone)] -struct SwitchDiscriminantInfo<'tcx> { - /// Type of the discriminant being switched on - discr_ty: Ty<'tcx>, - /// The basic block that the otherwise branch points to - otherwise_bb: BasicBlock, - /// Target along with the value being branched from. Otherwise is not included - targets_with_values: Vec<(u128, BasicBlock)>, - discr_source_info: SourceInfo, - /// The place of the discriminant used in the switch - discr_used_in_switch: Place<'tcx>, - /// The place of the adt that has its discriminant read - place_of_adt_discr_read: Place<'tcx>, - /// The type of the adt that has its discriminant read - type_adt_matched_on: Ty<'tcx>, -} - -#[derive(Debug)] -struct OptimizationToApply<'tcx> { - infos: Vec<OptimizationInfo<'tcx>>, - /// Basic block of the original first switch - basic_block_first_switch: BasicBlock, +/// Returns true if computing the discriminant of `place` may be hoisted out of the branch +fn may_hoist<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, place: Place<'tcx>) -> bool { + for (place, proj) in place.iter_projections() { + match proj { + // Dereferencing in the computation of `place` might cause issues from one of two + // cateogires. First, the referrent might be invalid. We protect against this by + // dereferencing references only (not pointers). Second, the use of a reference may + // invalidate other references that are used later (for aliasing reasons). Consider + // where such an invalidated reference may appear: + // - In `Q`: Not possible since `Q` is used as the operand of a `SwitchInt` and so + // cannot contain referenced data. + // - In `BBU`: Not possible since that block contains only the `unreachable` terminator + // - In `BBC.2, BBD.2`: Not possible, since `discriminant(P)` was computed prior to + // reaching that block in the input to our transformation, and so any data + // invalidated by that computation could not have been used there. + // - In `BB9`: Not possible since control flow might have reached `BB9` via the + // `otherwise` branch in `BBC, BBD` in the input to our transformation, which would + // have invalidated the data when computing `discriminant(P)` + // So dereferencing here is correct. + ProjectionElem::Deref => match place.ty(body.local_decls(), tcx).ty.kind() { + ty::Ref(..) => {} + _ => return false, + }, + // Field projections are always valid + ProjectionElem::Field(..) => {} + // We cannot allow + // downcasts either, since the correctness of the downcast may depend on the parent + // branch being taken. An easy example of this is + // ``` + // Q = discriminant(_3) + // P = (_3 as Variant) + // ``` + // However, checking if the child and parent place are the same and only erroring then + // is not sufficient either, since the `discriminant(_3) == 1` (or whatever) check may + // be replaced by another optimization pass with any other condition that can be proven + // equivalent. + ProjectionElem::Downcast(..) => { + return false; + } + // We cannot allow indexing since the index may be out of bounds. + _ => { + return false; + } + } + } + true } #[derive(Debug)] -struct OptimizationInfo<'tcx> { - /// Info about the first switch and discriminant - first_switch_info: SwitchDiscriminantInfo<'tcx>, - /// Info about the second switch and discriminant - second_switch_info: SwitchDiscriminantInfo<'tcx>, +struct OptimizationData<'tcx> { + destination: BasicBlock, + child_place: Place<'tcx>, + child_ty: Ty<'tcx>, + child_source: SourceInfo, } -impl<'a, 'tcx> Helper<'a, 'tcx> { - pub fn go( - &self, - bb: &BasicBlockData<'tcx>, - switch: &Terminator<'tcx>, - ) -> Option<Vec<OptimizationInfo<'tcx>>> { - // try to find the statement that defines the discriminant that is used for the switch - let discr = self.find_switch_discriminant_info(bb, switch)?; - - // go through each target, finding a discriminant read, and a switch - let results = discr - .targets_with_values - .iter() - .map(|(value, target)| self.find_discriminant_switch_pairing(&discr, *target, *value)); - - // if the optimization did not apply for one of the targets, then abort - if results.clone().any(|x| x.is_none()) || results.len() == 0 { - trace!("NO: not all of the targets matched the pattern for optimization"); - return None; +fn evaluate_candidate<'tcx>( + tcx: TyCtxt<'tcx>, + body: &Body<'tcx>, + parent: BasicBlock, +) -> Option<OptimizationData<'tcx>> { + let bbs = body.basic_blocks(); + let TerminatorKind::SwitchInt { + targets, + switch_ty: parent_ty, + .. + } = &bbs[parent].terminator().kind else { + return None + }; + let parent_dest = { + let poss = targets.otherwise(); + // If the fallthrough on the parent is trivially unreachable, we can let the + // children choose the destination + if bbs[poss].statements.len() == 0 + && bbs[poss].terminator().kind == TerminatorKind::Unreachable + { + None + } else { + Some(poss) } - - Some(results.flatten().collect()) + }; + let Some((_, child)) = targets.iter().next() else { + return None + }; + let child_terminator = &bbs[child].terminator(); + let TerminatorKind::SwitchInt { + switch_ty: child_ty, + targets: child_targets, + .. + } = &child_terminator.kind else { + return None + }; + if child_ty != parent_ty { + return None; + } + let Some(StatementKind::Assign(boxed)) + = &bbs[child].statements.first().map(|x| &x.kind) else { + return None; + }; + let (_, Rvalue::Discriminant(child_place)) = &**boxed else { + return None; + }; + let destination = parent_dest.unwrap_or(child_targets.otherwise()); + + // Verify that the optimization is legal in general + // We can hoist evaluating the child discriminant out of the branch + if !may_hoist(tcx, body, *child_place) { + return None; } - fn find_discriminant_switch_pairing( - &self, - discr_info: &SwitchDiscriminantInfo<'tcx>, - target: BasicBlock, - value: u128, - ) -> Option<OptimizationInfo<'tcx>> { - let bb = &self.body.basic_blocks()[target]; - // find switch - let terminator = bb.terminator(); - if is_switch(terminator) { - let this_bb_discr_info = self.find_switch_discriminant_info(bb, terminator)?; - - // the types of the two adts matched on have to be equalfor this optimization to apply - if discr_info.type_adt_matched_on != this_bb_discr_info.type_adt_matched_on { - trace!( - "NO: types do not match. LHS: {:?}, RHS: {:?}", - discr_info.type_adt_matched_on, - this_bb_discr_info.type_adt_matched_on - ); - return None; - } - - // the otherwise branch of the two switches have to point to the same bb - if discr_info.otherwise_bb != this_bb_discr_info.otherwise_bb { - trace!("NO: otherwise target is not the same"); - return None; - } - - // check that the value being matched on is the same. The - if !this_bb_discr_info.targets_with_values.iter().any(|x| x.0 == value) { - trace!("NO: values being matched on are not the same"); - return None; - } - - // only allow optimization if the left and right of the tuple being matched are the same variants. - // so the following should not optimize - // ```rust - // let x: Option<()>; - // let y: Option<()>; - // match (x,y) { - // (Some(_), None) => {}, - // _ => {} - // } - // ``` - // We check this by seeing that the value of the first discriminant is the only other discriminant value being used as a target in the second switch - if !(this_bb_discr_info.targets_with_values.len() == 1 - && this_bb_discr_info.targets_with_values[0].0 == value) - { - trace!( - "NO: The second switch did not have only 1 target (besides otherwise) that had the same value as the value from the first switch that got us here" - ); - return None; - } - - // when the second place is a projection of the first one, it's not safe to calculate their discriminant values sequentially. - // for example, this should not be optimized: - // - // ```rust - // enum E<'a> { Empty, Some(&'a E<'a>), } - // let Some(Some(_)) = e; - // ``` - // - // ```mir - // bb0: { - // _2 = discriminant(*_1) - // switchInt(_2) -> [...] - // } - // bb1: { - // _3 = discriminant(*(((*_1) as Some).0: &E)) - // switchInt(_3) -> [...] - // } - // ``` - let discr_place = discr_info.place_of_adt_discr_read; - let this_discr_place = this_bb_discr_info.place_of_adt_discr_read; - if discr_place.local == this_discr_place.local - && this_discr_place.projection.starts_with(discr_place.projection) - { - trace!("NO: one target is the projection of another"); - return None; - } - - // if we reach this point, the optimization applies, and we should be able to optimize this case - // store the info that is needed to apply the optimization - - Some(OptimizationInfo { - first_switch_info: discr_info.clone(), - second_switch_info: this_bb_discr_info, - }) - } else { - None + // Verify that the optimization is legal for each branch + for (value, child) in targets.iter() { + if !verify_candidate_branch(&bbs[child], value, *child_place, destination) { + return None; } } + Some(OptimizationData { + destination, + child_place: *child_place, + child_ty, + child_source: child_terminator.source_info, + }) +} - fn find_switch_discriminant_info( - &self, - bb: &BasicBlockData<'tcx>, - switch: &Terminator<'tcx>, - ) -> Option<SwitchDiscriminantInfo<'tcx>> { - match &switch.kind { - TerminatorKind::SwitchInt { discr, targets, .. } => { - let discr_local = discr.place()?.as_local()?; - // the declaration of the discriminant read. Place of this read is being used in the switch - let discr_decl = &self.body.local_decls()[discr_local]; - let discr_ty = discr_decl.ty; - // the otherwise target lies as the last element - let otherwise_bb = targets.otherwise(); - let targets_with_values = targets.iter().collect(); - - // find the place of the adt where the discriminant is being read from - // assume this is the last statement of the block - let place_of_adt_discr_read = match bb.statements.last()?.kind { - StatementKind::Assign(box (_, Rvalue::Discriminant(adt_place))) => { - Some(adt_place) - } - _ => None, - }?; - - let type_adt_matched_on = place_of_adt_discr_read.ty(self.body, self.tcx).ty; - - Some(SwitchDiscriminantInfo { - discr_used_in_switch: discr.place()?, - discr_ty, - otherwise_bb, - targets_with_values, - discr_source_info: discr_decl.source_info, - place_of_adt_discr_read, - type_adt_matched_on, - }) - } - _ => unreachable!("must only be passed terminator that is a switch"), - } +fn verify_candidate_branch<'tcx>( + branch: &BasicBlockData<'tcx>, + value: u128, + place: Place<'tcx>, + destination: BasicBlock, +) -> bool { + // In order for the optimization to be correct, the branch must... + // ...have exactly one statement + if branch.statements.len() != 1 { + return false; + } + // ...assign the descriminant of `place` in that statement + let StatementKind::Assign(boxed) = &branch.statements[0].kind else { + return false + }; + let (discr_place, Rvalue::Discriminant(from_place)) = &**boxed else { + return false + }; + if *from_place != place { + return false; + } + // ...make that assignment to a local + if discr_place.projection.len() != 0 { + return false; + } + // ...terminate on a `SwitchInt` that invalidates that local + let TerminatorKind::SwitchInt{ discr: switch_op, targets, .. } = &branch.terminator().kind else { + return false + }; + if *switch_op != Operand::Move(*discr_place) { + return false; + } + // ...fall through to `destination` if the switch misses + if destination != targets.otherwise() { + return false; + } + // ...have a branch for value `value` + let mut iter = targets.iter(); + let Some((target_value, _)) = iter.next() else { + return false; + }; + if target_value != value { + return false; + } + // ...and have no more branches + if let Some(_) = iter.next() { + return false; } + return true; } diff --git a/compiler/rustc_mir_transform/src/elaborate_drops.rs b/compiler/rustc_mir_transform/src/elaborate_drops.rs index b9a48197a35..7320b2738a7 100644 --- a/compiler/rustc_mir_transform/src/elaborate_drops.rs +++ b/compiler/rustc_mir_transform/src/elaborate_drops.rs @@ -19,6 +19,10 @@ use std::fmt; pub struct ElaborateDrops; impl<'tcx> MirPass<'tcx> for ElaborateDrops { + fn phase_change(&self) -> Option<MirPhase> { + Some(MirPhase::DropLowering) + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { debug!("elaborate_drops({:?} @ {:?})", body.source, body.span); @@ -145,13 +149,13 @@ struct Elaborator<'a, 'b, 'tcx> { ctxt: &'a mut ElaborateDropsCtxt<'b, 'tcx>, } -impl<'a, 'b, 'tcx> fmt::Debug for Elaborator<'a, 'b, 'tcx> { +impl fmt::Debug for Elaborator<'_, '_, '_> { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { Ok(()) } } -impl<'a, 'b, 'tcx> DropElaborator<'a, 'tcx> for Elaborator<'a, 'b, 'tcx> { +impl<'a, 'tcx> DropElaborator<'a, 'tcx> for Elaborator<'a, '_, 'tcx> { type Path = MovePathIndex; fn patch(&mut self) -> &mut MirPatch<'tcx> { @@ -312,12 +316,12 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { LookupResult::Parent(Some(parent)) => { let (_maybe_live, maybe_dead) = self.init_data.maybe_live_dead(parent); if maybe_dead { - span_bug!( + self.tcx.sess.delay_span_bug( terminator.source_info.span, - "drop of untracked, uninitialized value {:?}, place {:?} ({:?})", - bb, - place, - path + &format!( + "drop of untracked, uninitialized value {:?}, place {:?} ({:?})", + bb, place, path, + ), ); } continue; @@ -364,10 +368,9 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { bb, ), LookupResult::Parent(..) => { - span_bug!( + self.tcx.sess.delay_span_bug( terminator.source_info.span, - "drop of untracked value {:?}", - bb + &format!("drop of untracked value {:?}", bb), ); } } diff --git a/compiler/rustc_mir_transform/src/function_item_references.rs b/compiler/rustc_mir_transform/src/function_item_references.rs index 996c158c062..1b9fddec2be 100644 --- a/compiler/rustc_mir_transform/src/function_item_references.rs +++ b/compiler/rustc_mir_transform/src/function_item_references.rs @@ -1,3 +1,4 @@ +use itertools::Itertools; use rustc_errors::Applicability; use rustc_hir::def_id::DefId; use rustc_middle::mir::visit::Visitor; @@ -11,12 +12,12 @@ use rustc_session::lint::builtin::FUNCTION_ITEM_REFERENCES; use rustc_span::{symbol::sym, Span}; use rustc_target::spec::abi::Abi; -use crate::MirPass; +use crate::MirLint; pub struct FunctionItemReferences; -impl<'tcx> MirPass<'tcx> for FunctionItemReferences { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { +impl<'tcx> MirLint<'tcx> for FunctionItemReferences { + fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) { let mut checker = FunctionItemRefChecker { tcx, body }; checker.visit_body(&body); } @@ -27,7 +28,7 @@ struct FunctionItemRefChecker<'a, 'tcx> { body: &'a Body<'tcx>, } -impl<'a, 'tcx> Visitor<'tcx> for FunctionItemRefChecker<'a, 'tcx> { +impl<'tcx> Visitor<'tcx> for FunctionItemRefChecker<'_, 'tcx> { /// Emits a lint for function reference arguments bound by `fmt::Pointer` or passed to /// `transmute`. This only handles arguments in calls outside macro expansions to avoid double /// counting function references formatted as pointers by macros. @@ -42,57 +43,31 @@ impl<'a, 'tcx> Visitor<'tcx> for FunctionItemRefChecker<'a, 'tcx> { } = &terminator.kind { let source_info = *self.body.source_info(location); - // Only handle function calls outside macros - if !source_info.span.from_expansion() { - let func_ty = func.ty(self.body, self.tcx); - if let ty::FnDef(def_id, substs_ref) = *func_ty.kind() { - // Handle calls to `transmute` - if self.tcx.is_diagnostic_item(sym::transmute, def_id) { - let arg_ty = args[0].ty(self.body, self.tcx); - for generic_inner_ty in arg_ty.walk(self.tcx) { - if let GenericArgKind::Type(inner_ty) = generic_inner_ty.unpack() { - if let Some((fn_id, fn_substs)) = - FunctionItemRefChecker::is_fn_ref(inner_ty) - { - let span = self.nth_arg_span(&args, 0); - self.emit_lint(fn_id, fn_substs, source_info, span); - } + let func_ty = func.ty(self.body, self.tcx); + if let ty::FnDef(def_id, substs_ref) = *func_ty.kind() { + // Handle calls to `transmute` + if self.tcx.is_diagnostic_item(sym::transmute, def_id) { + let arg_ty = args[0].ty(self.body, self.tcx); + for generic_inner_ty in arg_ty.walk() { + if let GenericArgKind::Type(inner_ty) = generic_inner_ty.unpack() { + if let Some((fn_id, fn_substs)) = + FunctionItemRefChecker::is_fn_ref(inner_ty) + { + let span = self.nth_arg_span(&args, 0); + self.emit_lint(fn_id, fn_substs, source_info, span); } } - } else { - self.check_bound_args(def_id, substs_ref, &args, source_info); } + } else { + self.check_bound_args(def_id, substs_ref, &args, source_info); } } } self.super_terminator(terminator, location); } - - /// Emits a lint for function references formatted with `fmt::Pointer::fmt` by macros. These - /// cases are handled as operands instead of call terminators to avoid any dependence on - /// unstable, internal formatting details like whether `fmt` is called directly or not. - fn visit_operand(&mut self, operand: &Operand<'tcx>, location: Location) { - let source_info = *self.body.source_info(location); - if source_info.span.from_expansion() { - let op_ty = operand.ty(self.body, self.tcx); - if let ty::FnDef(def_id, substs_ref) = *op_ty.kind() { - if self.tcx.is_diagnostic_item(sym::pointer_trait_fmt, def_id) { - let param_ty = substs_ref.type_at(0); - if let Some((fn_id, fn_substs)) = FunctionItemRefChecker::is_fn_ref(param_ty) { - // The operand's ctxt wouldn't display the lint since it's inside a macro so - // we have to use the callsite's ctxt. - let callsite_ctxt = source_info.span.source_callsite().ctxt(); - let span = source_info.span.with_ctxt(callsite_ctxt); - self.emit_lint(fn_id, fn_substs, source_info, span); - } - } - } - } - self.super_operand(operand, location); - } } -impl<'a, 'tcx> FunctionItemRefChecker<'a, 'tcx> { +impl<'tcx> FunctionItemRefChecker<'_, 'tcx> { /// Emits a lint for function reference arguments bound by `fmt::Pointer` in calls to the /// function defined by `def_id` with the substitutions `substs_ref`. fn check_bound_args( @@ -110,7 +85,7 @@ impl<'a, 'tcx> FunctionItemRefChecker<'a, 'tcx> { let arg_defs = self.tcx.fn_sig(def_id).skip_binder().inputs(); for (arg_num, arg_def) in arg_defs.iter().enumerate() { // For all types reachable from the argument type in the fn sig - for generic_inner_ty in arg_def.walk(self.tcx) { + for generic_inner_ty in arg_def.walk() { if let GenericArgKind::Type(inner_ty) = generic_inner_ty.unpack() { // If the inner type matches the type bound by `Pointer` if TyS::same_type(inner_ty, bound_ty) { @@ -119,7 +94,13 @@ impl<'a, 'tcx> FunctionItemRefChecker<'a, 'tcx> { if let Some((fn_id, fn_substs)) = FunctionItemRefChecker::is_fn_ref(subst_ty) { - let span = self.nth_arg_span(args, arg_num); + let mut span = self.nth_arg_span(args, arg_num); + if span.from_expansion() { + // The operand's ctxt wouldn't display the lint since it's inside a macro so + // we have to use the callsite's ctxt. + let callsite_ctxt = span.source_callsite().ctxt(); + span = span.with_ctxt(callsite_ctxt); + } self.emit_lint(fn_id, fn_substs, source_info, span); } } @@ -197,7 +178,7 @@ impl<'a, 'tcx> FunctionItemRefChecker<'a, 'tcx> { let ident = self.tcx.item_name(fn_id).to_ident_string(); let ty_params = fn_substs.types().map(|ty| format!("{}", ty)); let const_params = fn_substs.consts().map(|c| format!("{}", c)); - let params = ty_params.chain(const_params).collect::<Vec<String>>().join(", "); + let params = ty_params.chain(const_params).join(", "); let num_args = fn_sig.inputs().map_bound(|inputs| inputs.len()).skip_binder(); let variadic = if fn_sig.c_variadic() { ", ..." } else { "" }; let ret = if fn_sig.output().skip_binder().is_unit() { "" } else { " -> _" }; diff --git a/compiler/rustc_mir_transform/src/generator.rs b/compiler/rustc_mir_transform/src/generator.rs index bc72e9d94a9..433a1c6ad67 100644 --- a/compiler/rustc_mir_transform/src/generator.rs +++ b/compiler/rustc_mir_transform/src/generator.rs @@ -233,7 +233,7 @@ struct TransformVisitor<'tcx> { new_ret_local: Local, } -impl TransformVisitor<'tcx> { +impl<'tcx> TransformVisitor<'tcx> { // Make a GeneratorState variant assignment. `core::ops::GeneratorState` only has single // element tuple variants, so we can just write to the downcasted first field and then set the // discriminant to the appropriate variant. @@ -243,7 +243,7 @@ impl TransformVisitor<'tcx> { val: Operand<'tcx>, source_info: SourceInfo, ) -> impl Iterator<Item = Statement<'tcx>> { - let kind = AggregateKind::Adt(self.state_adt_ref, idx, self.state_substs, None, None); + let kind = AggregateKind::Adt(self.state_adt_ref.did, idx, self.state_substs, None, None); assert_eq!(self.state_adt_ref.variants[idx].fields.len(), 1); let ty = self .tcx @@ -295,7 +295,7 @@ impl TransformVisitor<'tcx> { } } -impl MutVisitor<'tcx> for TransformVisitor<'tcx> { +impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } @@ -446,7 +446,7 @@ struct LivenessInfo { storage_liveness: IndexVec<BasicBlock, Option<BitSet<Local>>>, } -fn locals_live_across_suspend_points( +fn locals_live_across_suspend_points<'tcx>( tcx: TyCtxt<'tcx>, body: &Body<'tcx>, always_live_locals: &storage::AlwaysLiveLocals, @@ -613,7 +613,7 @@ impl ops::Deref for GeneratorSavedLocals { /// time. Generates a bitset for every local of all the other locals that may be /// StorageLive simultaneously with that local. This is used in the layout /// computation; see `GeneratorLayout` for more. -fn compute_storage_conflicts( +fn compute_storage_conflicts<'mir, 'tcx>( body: &'mir Body<'tcx>, saved_locals: &GeneratorSavedLocals, always_live_locals: storage::AlwaysLiveLocals, @@ -672,7 +672,9 @@ struct StorageConflictVisitor<'mir, 'tcx, 's> { local_conflicts: BitMatrix<Local, Local>, } -impl rustc_mir_dataflow::ResultsVisitor<'mir, 'tcx> for StorageConflictVisitor<'mir, 'tcx, '_> { +impl<'mir, 'tcx> rustc_mir_dataflow::ResultsVisitor<'mir, 'tcx> + for StorageConflictVisitor<'mir, 'tcx, '_> +{ type FlowState = BitSet<Local>; fn visit_statement_before_primary_effect( @@ -694,7 +696,7 @@ impl rustc_mir_dataflow::ResultsVisitor<'mir, 'tcx> for StorageConflictVisitor<' } } -impl<'body, 'tcx, 's> StorageConflictVisitor<'body, 'tcx, 's> { +impl StorageConflictVisitor<'_, '_, '_> { fn apply_state(&mut self, flow_state: &BitSet<Local>, loc: Location) { // Ignore unreachable blocks. if self.body.basic_blocks()[loc.block].terminator().kind == TerminatorKind::Unreachable { @@ -724,9 +726,13 @@ fn sanitize_witness<'tcx>( saved_locals: &GeneratorSavedLocals, ) { let did = body.source.def_id(); - let allowed_upvars = tcx.erase_regions(upvars); + let param_env = tcx.param_env(did); + + let allowed_upvars = tcx.normalize_erasing_regions(param_env, upvars); let allowed = match witness.kind() { - &ty::GeneratorWitness(s) => tcx.erase_late_bound_regions(s), + &ty::GeneratorWitness(interior_tys) => { + tcx.normalize_erasing_late_bound_regions(param_env, interior_tys) + } _ => { tcx.sess.delay_span_bug( body.span, @@ -736,8 +742,6 @@ fn sanitize_witness<'tcx>( } }; - let param_env = tcx.param_env(did); - for (local, decl) in body.local_decls.iter_enumerated() { // Ignore locals which are internal or not saved between yields. if !saved_locals.contains(local) || decl.internal { @@ -1232,6 +1236,10 @@ fn create_cases<'tcx>( } impl<'tcx> MirPass<'tcx> for StateTransform { + fn phase_change(&self) -> Option<MirPhase> { + Some(MirPhase::GeneratorLowering) + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let yield_ty = if let Some(yield_ty) = body.yield_ty() { yield_ty @@ -1394,7 +1402,7 @@ impl EnsureGeneratorFieldAssignmentsNeverAlias<'_> { self.saved_locals.get(place.local) } - fn check_assigned_place(&mut self, place: Place<'tcx>, f: impl FnOnce(&mut Self)) { + fn check_assigned_place(&mut self, place: Place<'_>, f: impl FnOnce(&mut Self)) { if let Some(assigned_local) = self.saved_local_for_direct_place(place) { assert!(self.assigned_local.is_none(), "`check_assigned_place` must not recurse"); @@ -1405,7 +1413,7 @@ impl EnsureGeneratorFieldAssignmentsNeverAlias<'_> { } } -impl Visitor<'tcx> for EnsureGeneratorFieldAssignmentsNeverAlias<'_> { +impl<'tcx> Visitor<'tcx> for EnsureGeneratorFieldAssignmentsNeverAlias<'_> { fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, location: Location) { let lhs = match self.assigned_local { Some(l) => l, @@ -1441,9 +1449,6 @@ impl Visitor<'tcx> for EnsureGeneratorFieldAssignmentsNeverAlias<'_> { self.check_assigned_place(*lhs, |this| this.visit_rvalue(rhs, location)); } - // FIXME: Does `llvm_asm!` have any aliasing requirements? - StatementKind::LlvmInlineAsm(_) => {} - StatementKind::FakeRead(..) | StatementKind::SetDiscriminant { .. } | StatementKind::StorageLive(_) diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 84a1e3fb600..e1f30fef44f 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -37,21 +37,16 @@ struct CallSite<'tcx> { source_info: SourceInfo, } -/// Returns true if MIR inlining is enabled in the current compilation session. -crate fn is_enabled(tcx: TyCtxt<'_>) -> bool { - if let Some(enabled) = tcx.sess.opts.debugging_opts.inline_mir { - return enabled; - } - - tcx.sess.mir_opt_level() >= 3 -} - impl<'tcx> MirPass<'tcx> for Inline { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if !is_enabled(tcx) { - return; + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + if let Some(enabled) = sess.opts.debugging_opts.inline_mir { + return enabled; } + sess.opts.mir_opt_level() >= 3 + } + + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let span = trace_span!("inline", body = %tcx.def_path_str(body.source.def_id())); let _guard = span.enter(); if inline(tcx, body) { @@ -62,7 +57,7 @@ impl<'tcx> MirPass<'tcx> for Inline { } } -fn inline(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { +fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { let def_id = body.source.def_id(); let hir_id = tcx.hir().local_def_id_to_hir_id(def_id.expect_local()); @@ -73,6 +68,12 @@ fn inline(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { if body.source.promoted.is_some() { return false; } + // Avoid inlining into generators, since their `optimized_mir` is used for layout computation, + // which can create a cycle, even when no attempt is made to inline the function in the other + // direction. + if body.generator.is_some() { + return false; + } let mut this = Inliner { tcx, @@ -100,7 +101,7 @@ struct Inliner<'tcx> { changed: bool, } -impl Inliner<'tcx> { +impl<'tcx> Inliner<'tcx> { fn process_blocks(&mut self, caller_body: &mut Body<'tcx>, blocks: Range<BasicBlock>) { for bb in blocks { let bb_data = &caller_body[bb]; @@ -207,19 +208,12 @@ impl Inliner<'tcx> { if let Some(callee_def_id) = callee.def_id().as_local() { let callee_hir_id = self.tcx.hir().local_def_id_to_hir_id(callee_def_id); - // Avoid inlining into generators, - // since their `optimized_mir` is used for layout computation, which can - // create a cycle, even when no attempt is made to inline the function - // in the other direction. - if caller_body.generator.is_some() { - return Err("local generator (query cycle avoidance)"); - } - // Avoid a cycle here by only using `instance_mir` only if we have // a lower `HirId` than the callee. This ensures that the callee will // not inline us. This trick only works without incremental compilation. // So don't do it if that is enabled. - if !self.tcx.dep_graph.is_fully_enabled() && self.hir_id < callee_hir_id { + if !self.tcx.dep_graph.is_fully_enabled() && self.hir_id.index() < callee_hir_id.index() + { return Ok(()); } @@ -441,6 +435,13 @@ impl Inliner<'tcx> { } } TerminatorKind::Resume => cost += RESUME_PENALTY, + TerminatorKind::InlineAsm { cleanup, .. } => { + cost += INSTR_COST; + + if cleanup.is_some() { + cost += LANDINGPAD_PENALTY; + } + } _ => cost += INSTR_COST, } @@ -784,7 +785,7 @@ struct Integrator<'a, 'tcx> { always_live_locals: BitSet<Local>, } -impl<'a, 'tcx> Integrator<'a, 'tcx> { +impl Integrator<'_, '_> { fn map_local(&self, local: Local) -> Local { let new = if local == RETURN_PLACE { self.destination.local @@ -813,7 +814,7 @@ impl<'a, 'tcx> Integrator<'a, 'tcx> { } } -impl<'a, 'tcx> MutVisitor<'tcx> for Integrator<'a, 'tcx> { +impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } @@ -954,9 +955,13 @@ impl<'a, 'tcx> MutVisitor<'tcx> for Integrator<'a, 'tcx> { { bug!("False unwinds should have been removed before inlining") } - TerminatorKind::InlineAsm { ref mut destination, .. } => { + TerminatorKind::InlineAsm { ref mut destination, ref mut cleanup, .. } => { if let Some(ref mut tgt) = *destination { *tgt = self.map_block(*tgt); + } else if !self.in_cleanup_block { + // Unless this inline asm is in a cleanup block, add an unwind edge to + // the original call's cleanup block + *cleanup = self.cleanup_block; } } } diff --git a/compiler/rustc_mir_transform/src/inline/cycle.rs b/compiler/rustc_mir_transform/src/inline/cycle.rs index 385394ba67d..44ded1647fc 100644 --- a/compiler/rustc_mir_transform/src/inline/cycle.rs +++ b/compiler/rustc_mir_transform/src/inline/cycle.rs @@ -10,7 +10,7 @@ use rustc_session::Limit; // FIXME: check whether it is cheaper to precompute the entire call graph instead of invoking // this query riddiculously often. #[instrument(level = "debug", skip(tcx, root, target))] -crate fn mir_callgraph_reachable( +crate fn mir_callgraph_reachable<'tcx>( tcx: TyCtxt<'tcx>, (root, target): (ty::Instance<'tcx>, LocalDefId), ) -> bool { @@ -33,7 +33,7 @@ crate fn mir_callgraph_reachable( level = "debug", skip(tcx, param_env, target, stack, seen, recursion_limiter, caller, recursion_limit) )] - fn process( + fn process<'tcx>( tcx: TyCtxt<'tcx>, param_env: ty::ParamEnv<'tcx>, caller: ty::Instance<'tcx>, @@ -89,7 +89,7 @@ crate fn mir_callgraph_reachable( // FIXME: A not fully substituted drop shim can cause ICEs if one attempts to // have its MIR built. Likely oli-obk just screwed up the `ParamEnv`s, so this // needs some more analysis. - if callee.definitely_needs_subst(tcx) { + if callee.needs_subst() { continue; } } diff --git a/compiler/rustc_mir_transform/src/instcombine.rs b/compiler/rustc_mir_transform/src/instcombine.rs index e15a69c95ae..792ac68671e 100644 --- a/compiler/rustc_mir_transform/src/instcombine.rs +++ b/compiler/rustc_mir_transform/src/instcombine.rs @@ -11,6 +11,10 @@ use rustc_middle::ty::{self, TyCtxt}; pub struct InstCombine; impl<'tcx> MirPass<'tcx> for InstCombine { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() > 0 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let (basic_blocks, local_decls) = body.basic_blocks_and_local_decls_mut(); let ctx = InstCombineContext { tcx, local_decls }; @@ -34,7 +38,7 @@ struct InstCombineContext<'tcx, 'a> { local_decls: &'a LocalDecls<'tcx>, } -impl<'tcx, 'a> InstCombineContext<'tcx, 'a> { +impl<'tcx> InstCombineContext<'tcx, '_> { fn should_combine(&self, source_info: &SourceInfo, rvalue: &Rvalue<'tcx>) -> bool { self.tcx.consider_optimizing(|| { format!("InstCombine - Rvalue: {:?} SourceInfo: {:?}", rvalue, source_info) diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index f9ef3146278..8e1601fb719 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -1,8 +1,6 @@ #![feature(box_patterns)] #![feature(box_syntax)] #![feature(crate_visibility_modifier)] -#![feature(in_band_lifetimes)] -#![feature(iter_zip)] #![feature(let_else)] #![feature(map_try_insert)] #![feature(min_specialization)] @@ -20,18 +18,23 @@ extern crate rustc_middle; use required_consts::RequiredConstsVisitor; use rustc_const_eval::util; -use rustc_data_structures::fx::FxHashSet; +use rustc_data_structures::fx::FxIndexSet; use rustc_data_structures::steal::Steal; use rustc_hir as hir; use rustc_hir::def_id::{DefId, LocalDefId}; -use rustc_hir::intravisit::{self, NestedVisitorMap, Visitor}; +use rustc_hir::intravisit::{self, Visitor}; use rustc_index::vec::IndexVec; use rustc_middle::mir::visit::Visitor as _; -use rustc_middle::mir::{dump_mir, traversal, Body, ConstQualifs, MirPhase, Promoted}; +use rustc_middle::mir::{traversal, Body, ConstQualifs, MirPass, MirPhase, Promoted}; use rustc_middle::ty::query::Providers; use rustc_middle::ty::{self, TyCtxt, TypeFoldable}; use rustc_span::{Span, Symbol}; +#[macro_use] +mod pass_manager; + +use pass_manager::{self as pm, Lint, MirLint, WithMinOptLevel}; + mod abort_unwinding_calls; mod add_call_guards; mod add_moves_for_packed_drops; @@ -56,12 +59,15 @@ mod inline; mod instcombine; mod lower_intrinsics; mod lower_slice_len; +mod marker; mod match_branches; mod multiple_return_terminators; mod normalize_array_len; mod nrvo; +mod remove_false_edges; mod remove_noop_landing_pads; mod remove_storage_markers; +mod remove_uninit_drops; mod remove_unneeded_drops; mod remove_zsts; mod required_consts; @@ -75,10 +81,9 @@ mod simplify_try; mod uninhabited_enum_branching; mod unreachable_prop; -use rustc_const_eval::transform::check_consts; +use rustc_const_eval::transform::check_consts::{self, ConstCx}; use rustc_const_eval::transform::promote_consts; use rustc_const_eval::transform::validate; -pub use rustc_const_eval::transform::MirPass; use rustc_mir_dataflow::rustc_peek; pub fn provide(providers: &mut Providers) { @@ -131,8 +136,8 @@ fn is_mir_available(tcx: TyCtxt<'_>, def_id: DefId) -> bool { /// Finds the full set of `DefId`s within the current crate that have /// MIR associated with them. -fn mir_keys(tcx: TyCtxt<'_>, (): ()) -> FxHashSet<LocalDefId> { - let mut set = FxHashSet::default(); +fn mir_keys(tcx: TyCtxt<'_>, (): ()) -> FxIndexSet<LocalDefId> { + let mut set = FxIndexSet::default(); // All body-owners have MIR associated with them. set.extend(tcx.hir().body_owners()); @@ -141,9 +146,9 @@ fn mir_keys(tcx: TyCtxt<'_>, (): ()) -> FxHashSet<LocalDefId> { // they don't have a BodyId, so we need to build them separately. struct GatherCtors<'a, 'tcx> { tcx: TyCtxt<'tcx>, - set: &'a mut FxHashSet<LocalDefId>, + set: &'a mut FxIndexSet<LocalDefId>, } - impl<'a, 'tcx> Visitor<'tcx> for GatherCtors<'a, 'tcx> { + impl<'tcx> Visitor<'tcx> for GatherCtors<'_, 'tcx> { fn visit_variant_data( &mut self, v: &'tcx hir::VariantData<'tcx>, @@ -157,76 +162,12 @@ fn mir_keys(tcx: TyCtxt<'_>, (): ()) -> FxHashSet<LocalDefId> { } intravisit::walk_struct_def(self, v) } - type Map = intravisit::ErasedMap<'tcx>; - fn nested_visit_map(&mut self) -> NestedVisitorMap<Self::Map> { - NestedVisitorMap::None - } } tcx.hir().visit_all_item_likes(&mut GatherCtors { tcx, set: &mut set }.as_deep_visitor()); set } -fn run_passes( - tcx: TyCtxt<'tcx>, - body: &mut Body<'tcx>, - mir_phase: MirPhase, - passes: &[&[&dyn MirPass<'tcx>]], -) { - let phase_index = mir_phase.phase_index(); - let validate = tcx.sess.opts.debugging_opts.validate_mir; - - if body.phase >= mir_phase { - return; - } - - if validate { - validate::Validator { when: format!("input to phase {:?}", mir_phase), mir_phase } - .run_pass(tcx, body); - } - - let mut index = 0; - let mut run_pass = |pass: &dyn MirPass<'tcx>| { - let run_hooks = |body: &_, index, is_after| { - let disambiguator = if is_after { "after" } else { "before" }; - dump_mir( - tcx, - Some(&format_args!("{:03}-{:03}", phase_index, index)), - &pass.name(), - &disambiguator, - body, - |_, _| Ok(()), - ); - }; - run_hooks(body, index, false); - pass.run_pass(tcx, body); - run_hooks(body, index, true); - - if validate { - validate::Validator { - when: format!("after {} in phase {:?}", pass.name(), mir_phase), - mir_phase, - } - .run_pass(tcx, body); - } - - index += 1; - }; - - for pass_group in passes { - for pass in *pass_group { - run_pass(*pass); - } - } - - body.phase = mir_phase; - - if mir_phase == MirPhase::Optimization { - validate::Validator { when: format!("end of phase {:?}", mir_phase), mir_phase } - .run_pass(tcx, body); - } -} - fn mir_const_qualif(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> ConstQualifs { let const_kind = tcx.hir().body_const_context(def.did); @@ -278,25 +219,25 @@ fn mir_const<'tcx>( rustc_middle::mir::dump_mir(tcx, None, "mir_map", &0, &body, |_, _| Ok(())); - run_passes( + pm::run_passes( tcx, &mut body, - MirPhase::Const, - &[&[ + &[ // MIR-level lints. - &check_packed_ref::CheckPackedRef, - &check_const_item_mutation::CheckConstItemMutation, - &function_item_references::FunctionItemReferences, + &Lint(check_packed_ref::CheckPackedRef), + &Lint(check_const_item_mutation::CheckConstItemMutation), + &Lint(function_item_references::FunctionItemReferences), // What we need to do constant evaluation. &simplify::SimplifyCfg::new("initial"), - &rustc_peek::SanityCheck, - ]], + &rustc_peek::SanityCheck, // Just a lint + &marker::PhaseChange(MirPhase::Const), + ], ); tcx.alloc_steal_mir(body) } /// Compute the main MIR body and the list of MIR bodies of the promoteds. -fn mir_promoted( +fn mir_promoted<'tcx>( tcx: TyCtxt<'tcx>, def: ty::WithOptConstParam<LocalDefId>, ) -> (&'tcx Steal<Body<'tcx>>, &'tcx Steal<IndexVec<Promoted, Body<'tcx>>>) { @@ -317,17 +258,17 @@ fn mir_promoted( } body.required_consts = required_consts; + // What we need to run borrowck etc. let promote_pass = promote_consts::PromoteTemps::default(); - let promote: &[&dyn MirPass<'tcx>] = &[ - // What we need to run borrowck etc. - &promote_pass, - &simplify::SimplifyCfg::new("promote-consts"), - ]; - - let opt_coverage: &[&dyn MirPass<'tcx>] = - if tcx.sess.instrument_coverage() { &[&coverage::InstrumentCoverage] } else { &[] }; - - run_passes(tcx, &mut body, MirPhase::ConstPromotion, &[promote, opt_coverage]); + pm::run_passes( + tcx, + &mut body, + &[ + &promote_pass, + &simplify::SimplifyCfg::new("promote-consts"), + &coverage::InstrumentCoverage, + ], + ); let promoted = promote_pass.promoted_fragments.into_inner(); (tcx.alloc_steal_mir(body), tcx.alloc_steal_promoted(promoted)) @@ -389,24 +330,15 @@ fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) - // Technically we want to not run on regular const items, but oli-obk doesn't know how to // conveniently detect that at this point without looking at the HIR. hir::ConstContext::Const => { - #[rustfmt::skip] - let optimizations: &[&dyn MirPass<'_>] = &[ - &const_prop::ConstProp, - ]; - - #[rustfmt::skip] - run_passes( + pm::run_passes( tcx, &mut body, - MirPhase::Optimization, - &[ - optimizations, - ], + &[&const_prop::ConstProp, &marker::PhaseChange(MirPhase::Optimization)], ); } } - debug_assert!(!body.has_free_regions(tcx), "Free regions in MIR for CTFE"); + debug_assert!(!body.has_free_regions(), "Free regions in MIR for CTFE"); body } @@ -430,14 +362,13 @@ fn mir_drops_elaborated_and_const_checked<'tcx>( tcx.ensure().mir_borrowck(def.did); } - let hir_id = tcx.hir().local_def_id_to_hir_id(def.did); - let is_fn_like = tcx.hir().get(hir_id).fn_kind().is_some(); + let is_fn_like = tcx.hir().get_by_def_id(def.did).fn_kind().is_some(); if is_fn_like { let did = def.did.to_def_id(); let def = ty::WithOptConstParam::unknown(did); // Do not compute the mir call graph without said call graph actually being used. - if inline::is_enabled(tcx) { + if inline::Inline.is_enabled(&tcx.sess) { let _ = tcx.mir_inliner_callees(ty::InstanceDef::Item(def)); } } @@ -445,8 +376,24 @@ fn mir_drops_elaborated_and_const_checked<'tcx>( let (body, _) = tcx.mir_promoted(def); let mut body = body.steal(); + // IMPORTANT + pm::run_passes(tcx, &mut body, &[&remove_false_edges::RemoveFalseEdges]); + + // Do a little drop elaboration before const-checking if `const_precise_live_drops` is enabled. + if check_consts::post_drop_elaboration::checking_enabled(&ConstCx::new(tcx, &body)) { + pm::run_passes( + tcx, + &mut body, + &[ + &simplify::SimplifyCfg::new("remove-false-edges"), + &remove_uninit_drops::RemoveUninitDrops, + ], + ); + check_consts::post_drop_elaboration::check_live_drops(tcx, &body); // FIXME: make this a MIR lint + } + run_post_borrowck_cleanup_passes(tcx, &mut body); - check_consts::post_drop_elaboration::check_live_drops(tcx, &body); + assert!(body.phase == MirPhase::DropLowering); tcx.alloc_steal_mir(body) } @@ -456,7 +403,7 @@ fn run_post_borrowck_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tc let post_borrowck_cleanup: &[&dyn MirPass<'tcx>] = &[ // Remove all things only needed by analysis - &simplify_branches::SimplifyBranches::new("initial"), + &simplify_branches::SimplifyConstCondition::new("initial"), &remove_noop_landing_pads::RemoveNoopLandingPads, &cleanup_post_borrowck::CleanupNonCodegenStatements, &simplify::SimplifyCfg::new("early-opt"), @@ -480,95 +427,72 @@ fn run_post_borrowck_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tc &deaggregator::Deaggregator, ]; - run_passes(tcx, body, MirPhase::DropLowering, &[post_borrowck_cleanup]); + pm::run_passes(tcx, body, post_borrowck_cleanup); } fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let mir_opt_level = tcx.sess.mir_opt_level(); + fn o1<T>(x: T) -> WithMinOptLevel<T> { + WithMinOptLevel(1, x) + } // Lowering generator control-flow and variables has to happen before we do anything else // to them. We run some optimizations before that, because they may be harder to do on the state // machine than on MIR with async primitives. - let optimizations_with_generators: &[&dyn MirPass<'tcx>] = &[ - &reveal_all::RevealAll, // has to be done before inlining, since inlined code is in RevealAll mode. - &lower_slice_len::LowerSliceLenCalls, // has to be done before inlining, otherwise actual call will be almost always inlined. Also simple, so can just do first - &normalize_array_len::NormalizeArrayLen, // has to run after `slice::len` lowering - &unreachable_prop::UnreachablePropagation, - &uninhabited_enum_branching::UninhabitedEnumBranching, - &simplify::SimplifyCfg::new("after-uninhabited-enum-branching"), - &inline::Inline, - &generator::StateTransform, - ]; - - // Even if we don't do optimizations, we still have to lower generators for codegen. - let no_optimizations_with_generators: &[&dyn MirPass<'tcx>] = &[&generator::StateTransform]; - - // The main optimizations that we do on MIR. - let optimizations: &[&dyn MirPass<'tcx>] = &[ - &remove_storage_markers::RemoveStorageMarkers, - &remove_zsts::RemoveZsts, - &const_goto::ConstGoto, - &remove_unneeded_drops::RemoveUnneededDrops, - &match_branches::MatchBranchSimplification, - // inst combine is after MatchBranchSimplification to clean up Ne(_1, false) - &multiple_return_terminators::MultipleReturnTerminators, - &instcombine::InstCombine, - &separate_const_switch::SeparateConstSwitch, - &const_prop::ConstProp, - &simplify_branches::SimplifyBranches::new("after-const-prop"), - &early_otherwise_branch::EarlyOtherwiseBranch, - &simplify_comparison_integral::SimplifyComparisonIntegral, - &simplify_try::SimplifyArmIdentity, - &simplify_try::SimplifyBranchSame, - &dest_prop::DestinationPropagation, - &simplify_branches::SimplifyBranches::new("final"), - &remove_noop_landing_pads::RemoveNoopLandingPads, - &simplify::SimplifyCfg::new("final"), - &nrvo::RenameReturnPlace, - &const_debuginfo::ConstDebugInfo, - &simplify::SimplifyLocals, - &multiple_return_terminators::MultipleReturnTerminators, - &deduplicate_blocks::DeduplicateBlocks, - ]; - - // Optimizations to run even if mir optimizations have been disabled. - let no_optimizations: &[&dyn MirPass<'tcx>] = &[ - // FIXME(#70073): This pass is responsible for both optimization as well as some lints. - &const_prop::ConstProp, - ]; - - // Some cleanup necessary at least for LLVM and potentially other codegen backends. - let pre_codegen_cleanup: &[&dyn MirPass<'tcx>] = &[ - &add_call_guards::CriticalCallEdges, - // Dump the end result for testing and debugging purposes. - &dump_mir::Marker("PreCodegen"), - ]; - - // End of pass declarations, now actually run the passes. - // Generator Lowering - #[rustfmt::skip] - run_passes( + pm::run_passes( tcx, body, - MirPhase::GeneratorLowering, &[ - if mir_opt_level > 0 { - optimizations_with_generators - } else { - no_optimizations_with_generators - } + &reveal_all::RevealAll, // has to be done before inlining, since inlined code is in RevealAll mode. + &lower_slice_len::LowerSliceLenCalls, // has to be done before inlining, otherwise actual call will be almost always inlined. Also simple, so can just do first + &normalize_array_len::NormalizeArrayLen, // has to run after `slice::len` lowering + &unreachable_prop::UnreachablePropagation, + &uninhabited_enum_branching::UninhabitedEnumBranching, + &o1(simplify::SimplifyCfg::new("after-uninhabited-enum-branching")), + &inline::Inline, + &generator::StateTransform, ], ); - // Main optimization passes - #[rustfmt::skip] - run_passes( + assert!(body.phase == MirPhase::GeneratorLowering); + + // The main optimizations that we do on MIR. + pm::run_passes( tcx, body, - MirPhase::Optimization, &[ - if mir_opt_level > 0 { optimizations } else { no_optimizations }, - pre_codegen_cleanup, + &remove_storage_markers::RemoveStorageMarkers, + &remove_zsts::RemoveZsts, + &const_goto::ConstGoto, + &remove_unneeded_drops::RemoveUnneededDrops, + &match_branches::MatchBranchSimplification, + // inst combine is after MatchBranchSimplification to clean up Ne(_1, false) + &multiple_return_terminators::MultipleReturnTerminators, + &instcombine::InstCombine, + &separate_const_switch::SeparateConstSwitch, + // + // FIXME(#70073): This pass is responsible for both optimization as well as some lints. + &const_prop::ConstProp, + // + // Const-prop runs unconditionally, but doesn't mutate the MIR at mir-opt-level=0. + &o1(simplify_branches::SimplifyConstCondition::new("after-const-prop")), + &early_otherwise_branch::EarlyOtherwiseBranch, + &simplify_comparison_integral::SimplifyComparisonIntegral, + &simplify_try::SimplifyArmIdentity, + &simplify_try::SimplifyBranchSame, + &dest_prop::DestinationPropagation, + &o1(simplify_branches::SimplifyConstCondition::new("final")), + &o1(remove_noop_landing_pads::RemoveNoopLandingPads), + &o1(simplify::SimplifyCfg::new("final")), + &nrvo::RenameReturnPlace, + &const_debuginfo::ConstDebugInfo, + &simplify::SimplifyLocals, + &multiple_return_terminators::MultipleReturnTerminators, + &deduplicate_blocks::DeduplicateBlocks, + // Some cleanup necessary at least for LLVM and potentially other codegen backends. + &add_call_guards::CriticalCallEdges, + &marker::PhaseChange(MirPhase::Optimization), + // Dump the end result for testing and debugging purposes. + &dump_mir::Marker("PreCodegen"), ], ); } @@ -601,7 +525,7 @@ fn inner_optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> Body<'_> { tcx.mir_drops_elaborated_and_const_checked(ty::WithOptConstParam::unknown(did)).steal(); run_optimization_passes(tcx, &mut body); - debug_assert!(!body.has_free_regions(tcx), "Free regions in optimized MIR"); + debug_assert!(!body.has_free_regions(), "Free regions in optimized MIR"); body } @@ -628,7 +552,7 @@ fn promoted_mir<'tcx>( run_post_borrowck_cleanup_passes(tcx, body); } - debug_assert!(!promoted.has_free_regions(tcx), "Free regions in promoted MIR"); + debug_assert!(!promoted.has_free_regions(), "Free regions in promoted MIR"); tcx.arena.alloc(promoted) } diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs index 5848163af72..4c4497ad629 100644 --- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs +++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs @@ -135,7 +135,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { } } -fn resolve_rust_intrinsic( +fn resolve_rust_intrinsic<'tcx>( tcx: TyCtxt<'tcx>, func_ty: Ty<'tcx>, ) -> Option<(Symbol, SubstsRef<'tcx>)> { @@ -148,7 +148,7 @@ fn resolve_rust_intrinsic( None } -fn validate_simd_shuffle(tcx: TyCtxt<'tcx>, args: &[Operand<'tcx>], span: Span) { +fn validate_simd_shuffle<'tcx>(tcx: TyCtxt<'tcx>, args: &[Operand<'tcx>], span: Span) { match &args[2] { Operand::Constant(_) => {} // all good _ => { diff --git a/compiler/rustc_mir_transform/src/lower_slice_len.rs b/compiler/rustc_mir_transform/src/lower_slice_len.rs index 822a372d8ce..c8297744873 100644 --- a/compiler/rustc_mir_transform/src/lower_slice_len.rs +++ b/compiler/rustc_mir_transform/src/lower_slice_len.rs @@ -10,6 +10,10 @@ use rustc_middle::ty::{self, TyCtxt}; pub struct LowerSliceLenCalls; impl<'tcx> MirPass<'tcx> for LowerSliceLenCalls { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.opts.mir_opt_level() > 0 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { lower_slice_len_calls(tcx, body) } diff --git a/compiler/rustc_mir_transform/src/marker.rs b/compiler/rustc_mir_transform/src/marker.rs new file mode 100644 index 00000000000..06819fc1d37 --- /dev/null +++ b/compiler/rustc_mir_transform/src/marker.rs @@ -0,0 +1,20 @@ +use std::borrow::Cow; + +use crate::MirPass; +use rustc_middle::mir::{Body, MirPhase}; +use rustc_middle::ty::TyCtxt; + +/// Changes the MIR phase without changing the MIR itself. +pub struct PhaseChange(pub MirPhase); + +impl<'tcx> MirPass<'tcx> for PhaseChange { + fn phase_change(&self) -> Option<MirPhase> { + Some(self.0) + } + + fn name(&self) -> Cow<'_, str> { + Cow::from(format!("PhaseChange-{:?}", self.0)) + } + + fn run_pass(&self, _: TyCtxt<'tcx>, _body: &mut Body<'tcx>) {} +} diff --git a/compiler/rustc_mir_transform/src/match_branches.rs b/compiler/rustc_mir_transform/src/match_branches.rs index c618abe9d05..3c14a324c36 100644 --- a/compiler/rustc_mir_transform/src/match_branches.rs +++ b/compiler/rustc_mir_transform/src/match_branches.rs @@ -40,11 +40,11 @@ pub struct MatchBranchSimplification; /// ``` impl<'tcx> MirPass<'tcx> for MatchBranchSimplification { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if tcx.sess.mir_opt_level() < 3 { - return; - } + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() >= 3 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let def_id = body.source.def_id(); let param_env = tcx.param_env(def_id); diff --git a/compiler/rustc_mir_transform/src/multiple_return_terminators.rs b/compiler/rustc_mir_transform/src/multiple_return_terminators.rs index b614917a883..22b6dead99c 100644 --- a/compiler/rustc_mir_transform/src/multiple_return_terminators.rs +++ b/compiler/rustc_mir_transform/src/multiple_return_terminators.rs @@ -9,11 +9,11 @@ use rustc_middle::ty::TyCtxt; pub struct MultipleReturnTerminators; impl<'tcx> MirPass<'tcx> for MultipleReturnTerminators { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if tcx.sess.mir_opt_level() < 4 { - return; - } + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() >= 4 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // find basic blocks with no statement and a return terminator let mut bbs_simple_returns = BitSet::new_empty(body.basic_blocks().len()); let def_id = body.source.def_id(); diff --git a/compiler/rustc_mir_transform/src/normalize_array_len.rs b/compiler/rustc_mir_transform/src/normalize_array_len.rs index a04a0b51531..e4ac57ac925 100644 --- a/compiler/rustc_mir_transform/src/normalize_array_len.rs +++ b/compiler/rustc_mir_transform/src/normalize_array_len.rs @@ -14,11 +14,11 @@ const MAX_NUM_LOCALS: usize = 3000; pub struct NormalizeArrayLen; impl<'tcx> MirPass<'tcx> for NormalizeArrayLen { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if tcx.sess.mir_opt_level() < 4 { - return; - } + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() >= 4 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // early returns for edge cases of highly unrolled functions if body.basic_blocks().len() > MAX_NUM_BLOCKS { return; @@ -85,7 +85,7 @@ struct Patcher<'a, 'tcx> { statement_idx: usize, } -impl<'a, 'tcx> Patcher<'a, 'tcx> { +impl<'tcx> Patcher<'_, 'tcx> { fn patch_expand_statement( &mut self, statement: &mut Statement<'tcx>, diff --git a/compiler/rustc_mir_transform/src/nrvo.rs b/compiler/rustc_mir_transform/src/nrvo.rs index 3ac4e77cf9a..797f7ee2685 100644 --- a/compiler/rustc_mir_transform/src/nrvo.rs +++ b/compiler/rustc_mir_transform/src/nrvo.rs @@ -33,11 +33,11 @@ use crate::MirPass; pub struct RenameReturnPlace; impl<'tcx> MirPass<'tcx> for RenameReturnPlace { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut mir::Body<'tcx>) { - if tcx.sess.mir_opt_level() == 0 { - return; - } + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() > 0 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut mir::Body<'tcx>) { let def_id = body.source.def_id(); let returned_local = match local_eligible_for_nrvo(body) { Some(l) => l, @@ -165,7 +165,7 @@ struct RenameToReturnPlace<'tcx> { } /// Replaces all uses of `self.to_rename` with `_0`. -impl MutVisitor<'tcx> for RenameToReturnPlace<'tcx> { +impl<'tcx> MutVisitor<'tcx> for RenameToReturnPlace<'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } @@ -221,7 +221,7 @@ impl IsReturnPlaceRead { } } -impl Visitor<'tcx> for IsReturnPlaceRead { +impl<'tcx> Visitor<'tcx> for IsReturnPlaceRead { fn visit_local(&mut self, &l: &Local, ctxt: PlaceContext, _: Location) { if l == mir::RETURN_PLACE && ctxt.is_use() && !ctxt.is_place_assignment() { self.0 = true; diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs new file mode 100644 index 00000000000..8725eae8709 --- /dev/null +++ b/compiler/rustc_mir_transform/src/pass_manager.rs @@ -0,0 +1,144 @@ +use std::borrow::Cow; + +use rustc_middle::mir::{self, Body, MirPhase}; +use rustc_middle::ty::TyCtxt; +use rustc_session::Session; + +use crate::{validate, MirPass}; + +/// Just like `MirPass`, except it cannot mutate `Body`. +pub trait MirLint<'tcx> { + fn name(&self) -> Cow<'_, str> { + let name = std::any::type_name::<Self>(); + if let Some(tail) = name.rfind(':') { + Cow::from(&name[tail + 1..]) + } else { + Cow::from(name) + } + } + + fn is_enabled(&self, _sess: &Session) -> bool { + true + } + + fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>); +} + +/// An adapter for `MirLint`s that implements `MirPass`. +#[derive(Debug, Clone)] +pub struct Lint<T>(pub T); + +impl<'tcx, T> MirPass<'tcx> for Lint<T> +where + T: MirLint<'tcx>, +{ + fn name(&self) -> Cow<'_, str> { + self.0.name() + } + + fn is_enabled(&self, sess: &Session) -> bool { + self.0.is_enabled(sess) + } + + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + self.0.run_lint(tcx, body) + } + + fn is_mir_dump_enabled(&self) -> bool { + false + } +} + +pub struct WithMinOptLevel<T>(pub u32, pub T); + +impl<'tcx, T> MirPass<'tcx> for WithMinOptLevel<T> +where + T: MirPass<'tcx>, +{ + fn name(&self) -> Cow<'_, str> { + self.1.name() + } + + fn is_enabled(&self, sess: &Session) -> bool { + sess.mir_opt_level() >= self.0 as usize + } + + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + self.1.run_pass(tcx, body) + } + + fn phase_change(&self) -> Option<MirPhase> { + self.1.phase_change() + } +} + +pub fn run_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, passes: &[&dyn MirPass<'tcx>]) { + let start_phase = body.phase; + let mut cnt = 0; + + let validate = tcx.sess.opts.debugging_opts.validate_mir; + + if validate { + validate_body(tcx, body, format!("start of phase transition from {:?}", start_phase)); + } + + for pass in passes { + if !pass.is_enabled(&tcx.sess) { + continue; + } + + let name = pass.name(); + let dump_enabled = pass.is_mir_dump_enabled(); + + if dump_enabled { + dump_mir(tcx, body, start_phase, &name, cnt, false); + } + + pass.run_pass(tcx, body); + + if dump_enabled { + dump_mir(tcx, body, start_phase, &name, cnt, true); + cnt += 1; + } + + if let Some(new_phase) = pass.phase_change() { + if body.phase >= new_phase { + panic!("Invalid MIR phase transition from {:?} to {:?}", body.phase, new_phase); + } + + body.phase = new_phase; + } + + if validate { + validate_body(tcx, body, format!("after pass {}", pass.name())); + } + } + + if validate || body.phase == MirPhase::Optimization { + validate_body(tcx, body, format!("end of phase transition to {:?}", body.phase)); + } +} + +pub fn validate_body<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, when: String) { + validate::Validator { when, mir_phase: body.phase }.run_pass(tcx, body); +} + +pub fn dump_mir<'tcx>( + tcx: TyCtxt<'tcx>, + body: &Body<'tcx>, + phase: MirPhase, + pass_name: &str, + cnt: usize, + is_after: bool, +) { + let phase_index = phase as u32; + + mir::dump_mir( + tcx, + Some(&format_args!("{:03}-{:03}", phase_index, cnt)), + pass_name, + if is_after { &"after" } else { &"before" }, + body, + |_, _| Ok(()), + ); +} diff --git a/compiler/rustc_mir_transform/src/remove_false_edges.rs b/compiler/rustc_mir_transform/src/remove_false_edges.rs new file mode 100644 index 00000000000..71f5ccf7e24 --- /dev/null +++ b/compiler/rustc_mir_transform/src/remove_false_edges.rs @@ -0,0 +1,29 @@ +use rustc_middle::mir::{Body, TerminatorKind}; +use rustc_middle::ty::TyCtxt; + +use crate::MirPass; + +/// Removes `FalseEdge` and `FalseUnwind` terminators from the MIR. +/// +/// These are only needed for borrow checking, and can be removed afterwards. +/// +/// FIXME: This should probably have its own MIR phase. +pub struct RemoveFalseEdges; + +impl<'tcx> MirPass<'tcx> for RemoveFalseEdges { + fn run_pass(&self, _: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + for block in body.basic_blocks_mut() { + let terminator = block.terminator_mut(); + terminator.kind = match terminator.kind { + TerminatorKind::FalseEdge { real_target, .. } => { + TerminatorKind::Goto { target: real_target } + } + TerminatorKind::FalseUnwind { real_target, .. } => { + TerminatorKind::Goto { target: real_target } + } + + _ => continue, + } + } + } +} diff --git a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs index 298bcd9dc24..77fb092d580 100644 --- a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs +++ b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs @@ -10,18 +10,14 @@ use rustc_target::spec::PanicStrategy; /// code for these. pub struct RemoveNoopLandingPads; -pub fn remove_noop_landing_pads<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if tcx.sess.panic_strategy() == PanicStrategy::Abort { - return; +impl<'tcx> MirPass<'tcx> for RemoveNoopLandingPads { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.panic_strategy() != PanicStrategy::Abort } - debug!("remove_noop_landing_pads({:?})", body); - - RemoveNoopLandingPads.remove_nop_landing_pads(body) -} -impl<'tcx> MirPass<'tcx> for RemoveNoopLandingPads { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - remove_noop_landing_pads(tcx, body); + fn run_pass(&self, _: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + debug!("remove_noop_landing_pads({:?})", body); + self.remove_nop_landing_pads(body) } } @@ -54,7 +50,6 @@ impl RemoveNoopLandingPads { StatementKind::Assign { .. } | StatementKind::SetDiscriminant { .. } - | StatementKind::LlvmInlineAsm { .. } | StatementKind::CopyNonOverlapping(..) | StatementKind::Retag { .. } => { return false; diff --git a/compiler/rustc_mir_transform/src/remove_storage_markers.rs b/compiler/rustc_mir_transform/src/remove_storage_markers.rs index 0c7323cbac5..c9b6e1459d3 100644 --- a/compiler/rustc_mir_transform/src/remove_storage_markers.rs +++ b/compiler/rustc_mir_transform/src/remove_storage_markers.rs @@ -7,6 +7,10 @@ use rustc_middle::ty::TyCtxt; pub struct RemoveStorageMarkers; impl<'tcx> MirPass<'tcx> for RemoveStorageMarkers { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() > 0 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { if tcx.sess.emit_lifetime_markers() { return; diff --git a/compiler/rustc_mir_transform/src/remove_uninit_drops.rs b/compiler/rustc_mir_transform/src/remove_uninit_drops.rs new file mode 100644 index 00000000000..fc5ac97e3e1 --- /dev/null +++ b/compiler/rustc_mir_transform/src/remove_uninit_drops.rs @@ -0,0 +1,171 @@ +use rustc_index::bit_set::BitSet; +use rustc_middle::mir::{Body, Field, Rvalue, Statement, StatementKind, TerminatorKind}; +use rustc_middle::ty::subst::SubstsRef; +use rustc_middle::ty::{self, ParamEnv, Ty, TyCtxt, VariantDef}; +use rustc_mir_dataflow::impls::MaybeInitializedPlaces; +use rustc_mir_dataflow::move_paths::{LookupResult, MoveData, MovePathIndex}; +use rustc_mir_dataflow::{self, move_path_children_matching, Analysis, MoveDataParamEnv}; + +use crate::MirPass; + +/// Removes `Drop` and `DropAndReplace` terminators whose target is known to be uninitialized at +/// that point. +/// +/// This is redundant with drop elaboration, but we need to do it prior to const-checking, and +/// running const-checking after drop elaboration makes it opimization dependent, causing issues +/// like [#90770]. +/// +/// [#90770]: https://github.com/rust-lang/rust/issues/90770 +pub struct RemoveUninitDrops; + +impl<'tcx> MirPass<'tcx> for RemoveUninitDrops { + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let param_env = tcx.param_env(body.source.def_id()); + let Ok(move_data) = MoveData::gather_moves(body, tcx, param_env) else { + // We could continue if there are move errors, but there's not much point since our + // init data isn't complete. + return; + }; + + let mdpe = MoveDataParamEnv { move_data, param_env }; + let mut maybe_inits = MaybeInitializedPlaces::new(tcx, body, &mdpe) + .into_engine(tcx, body) + .pass_name("remove_uninit_drops") + .iterate_to_fixpoint() + .into_results_cursor(body); + + let mut to_remove = vec![]; + for (bb, block) in body.basic_blocks().iter_enumerated() { + let terminator = block.terminator(); + let (TerminatorKind::Drop { place, .. } | TerminatorKind::DropAndReplace { place, .. }) + = &terminator.kind + else { continue }; + + maybe_inits.seek_before_primary_effect(body.terminator_loc(bb)); + + // If there's no move path for the dropped place, it's probably a `Deref`. Let it alone. + let LookupResult::Exact(mpi) = mdpe.move_data.rev_lookup.find(place.as_ref()) else { + continue; + }; + + let should_keep = is_needs_drop_and_init( + tcx, + param_env, + maybe_inits.get(), + &mdpe.move_data, + place.ty(body, tcx).ty, + mpi, + ); + if !should_keep { + to_remove.push(bb) + } + } + + for bb in to_remove { + let block = &mut body.basic_blocks_mut()[bb]; + + let (TerminatorKind::Drop { target, .. } | TerminatorKind::DropAndReplace { target, .. }) + = &block.terminator().kind + else { unreachable!() }; + + // Replace block terminator with `Goto`. + let target = *target; + let old_terminator_kind = std::mem::replace( + &mut block.terminator_mut().kind, + TerminatorKind::Goto { target }, + ); + + // If this is a `DropAndReplace`, we need to emulate the assignment to the return place. + if let TerminatorKind::DropAndReplace { place, value, .. } = old_terminator_kind { + block.statements.push(Statement { + source_info: block.terminator().source_info, + kind: StatementKind::Assign(Box::new((place, Rvalue::Use(value)))), + }); + } + } + } +} + +fn is_needs_drop_and_init<'tcx>( + tcx: TyCtxt<'tcx>, + param_env: ParamEnv<'tcx>, + maybe_inits: &BitSet<MovePathIndex>, + move_data: &MoveData<'tcx>, + ty: Ty<'tcx>, + mpi: MovePathIndex, +) -> bool { + // No need to look deeper if the root is definitely uninit or if it has no `Drop` impl. + if !maybe_inits.contains(mpi) || !ty.needs_drop(tcx, param_env) { + return false; + } + + let field_needs_drop_and_init = |(f, f_ty, mpi)| { + let child = move_path_children_matching(move_data, mpi, |x| x.is_field_to(f)); + let Some(mpi) = child else { + return f_ty.needs_drop(tcx, param_env); + }; + + is_needs_drop_and_init(tcx, param_env, maybe_inits, move_data, f_ty, mpi) + }; + + // This pass is only needed for const-checking, so it doesn't handle as many cases as + // `DropCtxt::open_drop`, since they aren't relevant in a const-context. + match ty.kind() { + ty::Adt(adt, substs) => { + let dont_elaborate = adt.is_union() || adt.is_manually_drop() || adt.has_dtor(tcx); + if dont_elaborate { + return true; + } + + // Look at all our fields, or if we are an enum all our variants and their fields. + // + // If a field's projection *is not* present in `MoveData`, it has the same + // initializedness as its parent (maybe init). + // + // If its projection *is* present in `MoveData`, then the field may have been moved + // from separate from its parent. Recurse. + adt.variants.iter_enumerated().any(|(vid, variant)| { + // Enums have multiple variants, which are discriminated with a `Downcast` projection. + // Structs have a single variant, and don't use a `Downcast` projection. + let mpi = if adt.is_enum() { + let downcast = + move_path_children_matching(move_data, mpi, |x| x.is_downcast_to(vid)); + let Some(dc_mpi) = downcast else { + return variant_needs_drop(tcx, param_env, substs, variant); + }; + + dc_mpi + } else { + mpi + }; + + variant + .fields + .iter() + .enumerate() + .map(|(f, field)| (Field::from_usize(f), field.ty(tcx, substs), mpi)) + .any(field_needs_drop_and_init) + }) + } + + ty::Tuple(_) => ty + .tuple_fields() + .enumerate() + .map(|(f, f_ty)| (Field::from_usize(f), f_ty, mpi)) + .any(field_needs_drop_and_init), + + _ => true, + } +} + +fn variant_needs_drop<'tcx>( + tcx: TyCtxt<'tcx>, + param_env: ParamEnv<'tcx>, + substs: SubstsRef<'tcx>, + variant: &VariantDef, +) -> bool { + variant.fields.iter().any(|field| { + let f_ty = field.ty(tcx, substs); + f_ty.needs_drop(tcx, param_env) + }) +} diff --git a/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs b/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs index c71bc512c31..39f78e9555e 100644 --- a/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs +++ b/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs @@ -1,4 +1,8 @@ -//! This pass replaces a drop of a type that does not need dropping, with a goto +//! This pass replaces a drop of a type that does not need dropping, with a goto. +//! +//! When the MIR is built, we check `needs_drop` before emitting a `Drop` for a place. This pass is +//! useful because (unlike MIR building) it runs after type checking, so it can make use of +//! `Reveal::All` to provide more precies type information. use crate::MirPass; use rustc_middle::mir::*; diff --git a/compiler/rustc_mir_transform/src/remove_zsts.rs b/compiler/rustc_mir_transform/src/remove_zsts.rs index d93ffa38c69..1d912e61409 100644 --- a/compiler/rustc_mir_transform/src/remove_zsts.rs +++ b/compiler/rustc_mir_transform/src/remove_zsts.rs @@ -8,6 +8,10 @@ use rustc_middle::ty::{self, Ty, TyCtxt}; pub struct RemoveZsts; impl<'tcx> MirPass<'tcx> for RemoveZsts { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() > 0 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // Avoid query cycles (generators require optimized MIR for layout). if tcx.type_of(body.source.def_id()).is_generator() { diff --git a/compiler/rustc_mir_transform/src/required_consts.rs b/compiler/rustc_mir_transform/src/required_consts.rs index 8b64ad65ab3..80c87cafea1 100644 --- a/compiler/rustc_mir_transform/src/required_consts.rs +++ b/compiler/rustc_mir_transform/src/required_consts.rs @@ -12,7 +12,7 @@ impl<'a, 'tcx> RequiredConstsVisitor<'a, 'tcx> { } } -impl<'a, 'tcx> Visitor<'tcx> for RequiredConstsVisitor<'a, 'tcx> { +impl<'tcx> Visitor<'tcx> for RequiredConstsVisitor<'_, 'tcx> { fn visit_constant(&mut self, constant: &Constant<'tcx>, _: Location) { if let Some(ct) = constant.literal.const_for_ty() { if let ConstKind::Unevaluated(_) = ct.val { diff --git a/compiler/rustc_mir_transform/src/reveal_all.rs b/compiler/rustc_mir_transform/src/reveal_all.rs index 6c423a2bb57..ee661793a44 100644 --- a/compiler/rustc_mir_transform/src/reveal_all.rs +++ b/compiler/rustc_mir_transform/src/reveal_all.rs @@ -8,15 +8,18 @@ use rustc_middle::ty::{self, Ty, TyCtxt}; pub struct RevealAll; impl<'tcx> MirPass<'tcx> for RevealAll { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.opts.mir_opt_level() >= 3 || super::inline::Inline.is_enabled(sess) + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - // This pass must run before inlining, since we insert callee bodies in RevealAll mode. // Do not apply this transformation to generators. - if (tcx.sess.mir_opt_level() >= 3 || super::inline::is_enabled(tcx)) - && body.generator.is_none() - { - let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); - RevealAllVisitor { tcx, param_env }.visit_body(body); + if body.generator.is_some() { + return; } + + let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); + RevealAllVisitor { tcx, param_env }.visit_body(body); } } @@ -33,26 +36,9 @@ impl<'tcx> MutVisitor<'tcx> for RevealAllVisitor<'tcx> { #[inline] fn visit_ty(&mut self, ty: &mut Ty<'tcx>, _: TyContext) { - *ty = self.tcx.normalize_erasing_regions(self.param_env, ty); - } - - #[inline] - fn process_projection_elem( - &mut self, - elem: PlaceElem<'tcx>, - _: Location, - ) -> Option<PlaceElem<'tcx>> { - match elem { - PlaceElem::Field(field, ty) => { - let new_ty = self.tcx.normalize_erasing_regions(self.param_env, ty); - if ty != new_ty { Some(PlaceElem::Field(field, new_ty)) } else { None } - } - // None of those contain a Ty. - PlaceElem::Index(..) - | PlaceElem::Deref - | PlaceElem::ConstantIndex { .. } - | PlaceElem::Subslice { .. } - | PlaceElem::Downcast(..) => None, - } + // We have to use `try_normalize_erasing_regions` here, since it's + // possible that we visit impossible-to-satisfy where clauses here, + // see #91745 + *ty = self.tcx.try_normalize_erasing_regions(self.param_env, *ty).unwrap_or(ty); } } diff --git a/compiler/rustc_mir_transform/src/separate_const_switch.rs b/compiler/rustc_mir_transform/src/separate_const_switch.rs index 3002e7041b0..d265720e182 100644 --- a/compiler/rustc_mir_transform/src/separate_const_switch.rs +++ b/compiler/rustc_mir_transform/src/separate_const_switch.rs @@ -45,11 +45,11 @@ use smallvec::SmallVec; pub struct SeparateConstSwitch; impl<'tcx> MirPass<'tcx> for SeparateConstSwitch { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if tcx.sess.mir_opt_level() < 4 { - return; - } + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() >= 4 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // If execution did something, applying a simplification layer // helps later passes optimize the copy away. if separate_const_switch(body) > 0 { @@ -59,7 +59,7 @@ impl<'tcx> MirPass<'tcx> for SeparateConstSwitch { } /// Returns the amount of blocks that were duplicated -pub fn separate_const_switch<'tcx>(body: &mut Body<'tcx>) -> usize { +pub fn separate_const_switch(body: &mut Body<'_>) -> usize { let mut new_blocks: SmallVec<[(BasicBlock, BasicBlock); 6]> = SmallVec::new(); let predecessors = body.predecessors(); 'block_iter: for (block_id, block) in body.basic_blocks().iter_enumerated() { @@ -239,10 +239,6 @@ fn is_likely_const<'tcx>(mut tracked_place: Place<'tcx>, block: &BasicBlockData< } } - // If inline assembly is found, we probably should - // not try to analyze the code - StatementKind::LlvmInlineAsm(_) => return false, - // These statements have no influence on the place // we are interested in StatementKind::FakeRead(_) @@ -320,10 +316,6 @@ fn find_determining_place<'tcx>( | StatementKind::CopyNonOverlapping(_) | StatementKind::Nop => {} - // If inline assembly is found, we probably should - // not try to analyze the code - StatementKind::LlvmInlineAsm(_) => return None, - // If the discriminant is set, it is always set // as a constant, so the job is already done. // As we are **ignoring projections**, if the place diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index f2ea5fedc62..919171db39e 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -17,8 +17,8 @@ use std::iter; use crate::util::expand_aggregate; use crate::{ - abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, remove_noop_landing_pads, - run_passes, simplify, + abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, marker, pass_manager as pm, + remove_noop_landing_pads, simplify, }; use rustc_middle::mir::patch::MirPatch; use rustc_mir_dataflow::elaborate_drops::{self, DropElaborator, DropFlagMode, DropStyle}; @@ -64,7 +64,19 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<' build_call_shim(tcx, instance, Some(Adjustment::RefMut), CallKind::Direct(call_mut)) } - ty::InstanceDef::DropGlue(def_id, ty) => build_drop_shim(tcx, def_id, ty), + + ty::InstanceDef::DropGlue(def_id, ty) => { + // FIXME(#91576): Drop shims for generators aren't subject to the MIR passes at the end + // of this function. Is this intentional? + if let Some(ty::Generator(gen_def_id, substs, _)) = ty.map(ty::TyS::kind) { + let body = tcx.optimized_mir(*gen_def_id).generator_drop().unwrap(); + let body = body.clone().subst(tcx, substs); + debug!("make_shim({:?}) = {:?}", instance, body); + return body; + } + + build_drop_shim(tcx, def_id, ty) + } ty::InstanceDef::CloneShim(def_id, ty) => build_clone_shim(tcx, def_id, ty), ty::InstanceDef::Virtual(..) => { bug!("InstanceDef::Virtual ({:?}) is for direct calls only", instance) @@ -75,17 +87,17 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<' }; debug!("make_shim({:?}) = untransformed {:?}", instance, result); - run_passes( + pm::run_passes( tcx, &mut result, - MirPhase::Const, - &[&[ + &[ &add_moves_for_packed_drops::AddMovesForPackedDrops, &remove_noop_landing_pads::RemoveNoopLandingPads, &simplify::SimplifyCfg::new("make_shim"), &add_call_guards::CriticalCallEdges, &abort_unwinding_calls::AbortUnwindingCalls, - ]], + &marker::PhaseChange(MirPhase::Const), + ], ); debug!("make_shim({:?}) = {:?}", instance, result); @@ -132,11 +144,7 @@ fn local_decls_for_sig<'tcx>( fn build_drop_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, ty: Option<Ty<'tcx>>) -> Body<'tcx> { debug!("build_drop_shim(def_id={:?}, ty={:?})", def_id, ty); - // Check if this is a generator, if so, return the drop glue for it - if let Some(&ty::Generator(gen_def_id, substs, _)) = ty.map(|ty| ty.kind()) { - let body = tcx.optimized_mir(gen_def_id).generator_drop().unwrap(); - return body.clone().subst(tcx, substs); - } + assert!(!matches!(ty, Some(ty) if ty.is_generator())); let substs = if let Some(ty) = ty { tcx.intern_substs(&[ty.into()]) @@ -163,7 +171,7 @@ fn build_drop_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, ty: Option<Ty<'tcx>>) let source = MirSource::from_instance(ty::InstanceDef::DropGlue(def_id, ty)); let mut body = - new_body(tcx, source, blocks, local_decls_for_sig(&sig, span), sig.inputs().len(), span); + new_body(source, blocks, local_decls_for_sig(&sig, span), sig.inputs().len(), span); if ty.is_some() { // The first argument (index 0), but add 1 for the return value. @@ -202,7 +210,6 @@ fn build_drop_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, ty: Option<Ty<'tcx>>) } fn new_body<'tcx>( - tcx: TyCtxt<'tcx>, source: MirSource<'tcx>, basic_blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>, local_decls: IndexVec<Local, LocalDecl<'tcx>>, @@ -210,7 +217,6 @@ fn new_body<'tcx>( span: Span, ) -> Body<'tcx> { Body::new( - tcx, source, basic_blocks, IndexVec::from_elem_n( @@ -239,7 +245,7 @@ pub struct DropShimElaborator<'a, 'tcx> { pub param_env: ty::ParamEnv<'tcx>, } -impl<'a, 'tcx> fmt::Debug for DropShimElaborator<'a, 'tcx> { +impl fmt::Debug for DropShimElaborator<'_, '_> { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { Ok(()) } @@ -310,7 +316,6 @@ fn build_clone_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) - match self_ty.kind() { _ if is_copy => builder.copy_shim(), - ty::Array(ty, len) => builder.array_shim(dest, src, ty, len), ty::Closure(_, substs) => { builder.tuple_like_shim(dest, src, substs.as_closure().upvar_tys()) } @@ -330,7 +335,7 @@ struct CloneShimBuilder<'tcx> { sig: ty::FnSig<'tcx>, } -impl CloneShimBuilder<'tcx> { +impl<'tcx> CloneShimBuilder<'tcx> { fn new(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Self { // we must subst the self_ty because it's // otherwise going to be TySelf and we can't index @@ -355,14 +360,7 @@ impl CloneShimBuilder<'tcx> { self.def_id, self.sig.inputs_and_output[0], )); - new_body( - self.tcx, - source, - self.blocks, - self.local_decls, - self.sig.inputs().len(), - self.span, - ) + new_body(source, self.blocks, self.local_decls, self.sig.inputs().len(), self.span) } fn source_info(&self) -> SourceInfo { @@ -459,154 +457,6 @@ impl CloneShimBuilder<'tcx> { ); } - fn loop_header( - &mut self, - beg: Place<'tcx>, - end: Place<'tcx>, - loop_body: BasicBlock, - loop_end: BasicBlock, - is_cleanup: bool, - ) { - let tcx = self.tcx; - - let cond = self.make_place(Mutability::Mut, tcx.types.bool); - let compute_cond = self.make_statement(StatementKind::Assign(Box::new(( - cond, - Rvalue::BinaryOp(BinOp::Ne, Box::new((Operand::Copy(end), Operand::Copy(beg)))), - )))); - - // `if end != beg { goto loop_body; } else { goto loop_end; }` - self.block( - vec![compute_cond], - TerminatorKind::if_(tcx, Operand::Move(cond), loop_body, loop_end), - is_cleanup, - ); - } - - fn make_usize(&self, value: u64) -> Box<Constant<'tcx>> { - Box::new(Constant { - span: self.span, - user_ty: None, - literal: ty::Const::from_usize(self.tcx, value).into(), - }) - } - - fn array_shim( - &mut self, - dest: Place<'tcx>, - src: Place<'tcx>, - ty: Ty<'tcx>, - len: &'tcx ty::Const<'tcx>, - ) { - let tcx = self.tcx; - let span = self.span; - - let beg = self.local_decls.push(LocalDecl::new(tcx.types.usize, span)); - let end = self.make_place(Mutability::Not, tcx.types.usize); - - // BB #0 - // `let mut beg = 0;` - // `let end = len;` - // `goto #1;` - let inits = vec![ - self.make_statement(StatementKind::Assign(Box::new(( - Place::from(beg), - Rvalue::Use(Operand::Constant(self.make_usize(0))), - )))), - self.make_statement(StatementKind::Assign(Box::new(( - end, - Rvalue::Use(Operand::Constant(Box::new(Constant { - span: self.span, - user_ty: None, - literal: len.into(), - }))), - )))), - ]; - self.block(inits, TerminatorKind::Goto { target: BasicBlock::new(1) }, false); - - // BB #1: loop { - // BB #2; - // BB #3; - // } - // BB #4; - self.loop_header(Place::from(beg), end, BasicBlock::new(2), BasicBlock::new(4), false); - - // BB #2 - // `dest[i] = Clone::clone(src[beg])`; - // Goto #3 if ok, #5 if unwinding happens. - let dest_field = self.tcx.mk_place_index(dest, beg); - let src_field = self.tcx.mk_place_index(src, beg); - self.make_clone_call(dest_field, src_field, ty, BasicBlock::new(3), BasicBlock::new(5)); - - // BB #3 - // `beg = beg + 1;` - // `goto #1`; - let statements = vec![self.make_statement(StatementKind::Assign(Box::new(( - Place::from(beg), - Rvalue::BinaryOp( - BinOp::Add, - Box::new((Operand::Copy(Place::from(beg)), Operand::Constant(self.make_usize(1)))), - ), - ))))]; - self.block(statements, TerminatorKind::Goto { target: BasicBlock::new(1) }, false); - - // BB #4 - // `return dest;` - self.block(vec![], TerminatorKind::Return, false); - - // BB #5 (cleanup) - // `let end = beg;` - // `let mut beg = 0;` - // goto #6; - let end = beg; - let beg = self.local_decls.push(LocalDecl::new(tcx.types.usize, span)); - let init = self.make_statement(StatementKind::Assign(Box::new(( - Place::from(beg), - Rvalue::Use(Operand::Constant(self.make_usize(0))), - )))); - self.block(vec![init], TerminatorKind::Goto { target: BasicBlock::new(6) }, true); - - // BB #6 (cleanup): loop { - // BB #7; - // BB #8; - // } - // BB #9; - self.loop_header( - Place::from(beg), - Place::from(end), - BasicBlock::new(7), - BasicBlock::new(9), - true, - ); - - // BB #7 (cleanup) - // `drop(dest[beg])`; - self.block( - vec![], - TerminatorKind::Drop { - place: self.tcx.mk_place_index(dest, beg), - target: BasicBlock::new(8), - unwind: None, - }, - true, - ); - - // BB #8 (cleanup) - // `beg = beg + 1;` - // `goto #6;` - let statement = self.make_statement(StatementKind::Assign(Box::new(( - Place::from(beg), - Rvalue::BinaryOp( - BinOp::Add, - Box::new((Operand::Copy(Place::from(beg)), Operand::Constant(self.make_usize(1)))), - ), - )))); - self.block(vec![statement], TerminatorKind::Goto { target: BasicBlock::new(6) }, true); - - // BB #9 (resume) - self.block(vec![], TerminatorKind::Resume, true); - } - fn tuple_like_shim<I>(&mut self, dest: Place<'tcx>, src: Place<'tcx>, tys: I) where I: Iterator<Item = Ty<'tcx>>, @@ -860,14 +710,8 @@ fn build_call_shim<'tcx>( block(&mut blocks, vec![], TerminatorKind::Resume, true); } - let mut body = new_body( - tcx, - MirSource::from_instance(instance), - blocks, - local_decls, - sig.inputs().len(), - span, - ); + let mut body = + new_body(MirSource::from_instance(instance), blocks, local_decls, sig.inputs().len(), span); if let Abi::RustCall = sig.abi { body.spread_arg = Some(Local::new(sig.inputs().len())); @@ -918,7 +762,7 @@ pub fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> Body<'_> { adt_def.variants[variant_index].fields.iter().enumerate().map(|(idx, field_def)| { (Operand::Move(Place::from(Local::new(idx + 1))), field_def.ty(tcx, substs)) }), - AggregateKind::Adt(adt_def, variant_index, substs, None, None), + AggregateKind::Adt(adt_def.did, variant_index, substs, None, None), source_info, tcx, ) @@ -932,7 +776,6 @@ pub fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> Body<'_> { let source = MirSource::item(ctor_id); let body = new_body( - tcx, source, IndexVec::from_elem_n(start_block, 1), local_decls, diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index d6cd505cbb5..7e0c8e233e9 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -47,7 +47,7 @@ impl SimplifyCfg { } } -pub fn simplify_cfg(tcx: TyCtxt<'tcx>, body: &mut Body<'_>) { +pub fn simplify_cfg<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { CfgSimplifier::new(body).simplify(); remove_dead_blocks(tcx, body); @@ -262,7 +262,7 @@ impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> { } } -pub fn remove_dead_blocks(tcx: TyCtxt<'tcx>, body: &mut Body<'_>) { +pub fn remove_dead_blocks<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let reachable = traversal::reachable_as_bitset(body); let num_blocks = body.basic_blocks().len(); if num_blocks == reachable.count() { @@ -368,6 +368,10 @@ fn save_unreachable_coverage( pub struct SimplifyLocals; impl<'tcx> MirPass<'tcx> for SimplifyLocals { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() > 0 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { trace!("running SimplifyLocals on {:?}", body.source); simplify_locals(body, tcx); @@ -450,7 +454,7 @@ impl UsedLocals { } /// Updates the use counts to reflect the removal of given statement. - fn statement_removed(&mut self, statement: &Statement<'tcx>) { + fn statement_removed(&mut self, statement: &Statement<'_>) { self.increment = false; // The location of the statement is irrelevant. @@ -459,7 +463,7 @@ impl UsedLocals { } /// Visits a left-hand side of an assignment. - fn visit_lhs(&mut self, place: &Place<'tcx>, location: Location) { + fn visit_lhs(&mut self, place: &Place<'_>, location: Location) { if place.is_indirect() { // A use, not a definition. self.visit_place(place, PlaceContext::MutatingUse(MutatingUseContext::Store), location); @@ -476,11 +480,10 @@ impl UsedLocals { } } -impl Visitor<'_> for UsedLocals { +impl<'tcx> Visitor<'tcx> for UsedLocals { fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { match statement.kind { - StatementKind::LlvmInlineAsm(..) - | StatementKind::CopyNonOverlapping(..) + StatementKind::CopyNonOverlapping(..) | StatementKind::Retag(..) | StatementKind::Coverage(..) | StatementKind::FakeRead(..) @@ -514,7 +517,7 @@ impl Visitor<'_> for UsedLocals { } /// Removes unused definitions. Updates the used locals to reflect the changes made. -fn remove_unused_definitions<'a, 'tcx>(used_locals: &'a mut UsedLocals, body: &mut Body<'tcx>) { +fn remove_unused_definitions(used_locals: &mut UsedLocals, body: &mut Body<'_>) { // The use counts are updated as we remove the statements. A local might become unused // during the retain operation, leading to a temporary inconsistency (storage statements or // definitions referencing the local might remain). For correctness it is crucial that this diff --git a/compiler/rustc_mir_transform/src/simplify_branches.rs b/compiler/rustc_mir_transform/src/simplify_branches.rs index df90cfa318d..3bbae5b8976 100644 --- a/compiler/rustc_mir_transform/src/simplify_branches.rs +++ b/compiler/rustc_mir_transform/src/simplify_branches.rs @@ -1,22 +1,21 @@ -//! A pass that simplifies branches when their condition is known. - use crate::MirPass; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; use std::borrow::Cow; -pub struct SimplifyBranches { +/// A pass that replaces a branch with a goto when its condition is known. +pub struct SimplifyConstCondition { label: String, } -impl SimplifyBranches { +impl SimplifyConstCondition { pub fn new(label: &str) -> Self { - SimplifyBranches { label: format!("SimplifyBranches-{}", label) } + SimplifyConstCondition { label: format!("SimplifyConstCondition-{}", label) } } } -impl<'tcx> MirPass<'tcx> for SimplifyBranches { +impl<'tcx> MirPass<'tcx> for SimplifyConstCondition { fn name(&self) -> Cow<'_, str> { Cow::Borrowed(&self.label) } @@ -34,15 +33,8 @@ impl<'tcx> MirPass<'tcx> for SimplifyBranches { } => { let constant = c.literal.try_eval_bits(tcx, param_env, switch_ty); if let Some(constant) = constant { - let otherwise = targets.otherwise(); - let mut ret = TerminatorKind::Goto { target: otherwise }; - for (v, t) in targets.iter() { - if v == constant { - ret = TerminatorKind::Goto { target: t }; - break; - } - } - ret + let target = targets.target_for_value(constant); + TerminatorKind::Goto { target } } else { continue; } @@ -53,12 +45,6 @@ impl<'tcx> MirPass<'tcx> for SimplifyBranches { Some(v) if v == expected => TerminatorKind::Goto { target }, _ => continue, }, - TerminatorKind::FalseEdge { real_target, .. } => { - TerminatorKind::Goto { target: real_target } - } - TerminatorKind::FalseUnwind { real_target, .. } => { - TerminatorKind::Goto { target: real_target } - } _ => continue, }; } diff --git a/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs b/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs index 948fcd9f455..da683a33651 100644 --- a/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs +++ b/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs @@ -26,6 +26,10 @@ use rustc_middle::{ pub struct SimplifyComparisonIntegral; impl<'tcx> MirPass<'tcx> for SimplifyComparisonIntegral { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() > 0 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { trace!("Running SimplifyComparisonIntegral on {:?}", body.source); @@ -144,7 +148,7 @@ struct OptimizationFinder<'a, 'tcx> { body: &'a Body<'tcx>, } -impl<'a, 'tcx> OptimizationFinder<'a, 'tcx> { +impl<'tcx> OptimizationFinder<'_, 'tcx> { fn find_optimizations(&self) -> Vec<OptimizationInfo<'tcx>> { self.body .basic_blocks() diff --git a/compiler/rustc_mir_transform/src/simplify_try.rs b/compiler/rustc_mir_transform/src/simplify_try.rs index e436d73226a..d5507fcc78c 100644 --- a/compiler/rustc_mir_transform/src/simplify_try.rs +++ b/compiler/rustc_mir_transform/src/simplify_try.rs @@ -102,7 +102,7 @@ fn get_arm_identity_info<'a, 'tcx>( type StmtIter<'a, 'tcx> = Peekable<Enumerate<Iter<'a, Statement<'tcx>>>>; - fn is_storage_stmt<'tcx>(stmt: &Statement<'tcx>) -> bool { + fn is_storage_stmt(stmt: &Statement<'_>) -> bool { matches!(stmt.kind, StatementKind::StorageLive(_) | StatementKind::StorageDead(_)) } @@ -122,8 +122,8 @@ fn get_arm_identity_info<'a, 'tcx>( /// Eats consecutive `StorageLive` and `StorageDead` Statements. /// The iterator `stmt_iter` is not advanced if none were found. - fn try_eat_storage_stmts<'a, 'tcx>( - stmt_iter: &mut StmtIter<'a, 'tcx>, + fn try_eat_storage_stmts( + stmt_iter: &mut StmtIter<'_, '_>, storage_live_stmts: &mut Vec<(usize, Local)>, storage_dead_stmts: &mut Vec<(usize, Local)>, ) { @@ -136,7 +136,7 @@ fn get_arm_identity_info<'a, 'tcx>( }) } - fn is_tmp_storage_stmt<'tcx>(stmt: &Statement<'tcx>) -> bool { + fn is_tmp_storage_stmt(stmt: &Statement<'_>) -> bool { use rustc_middle::mir::StatementKind::Assign; if let Assign(box (place, Rvalue::Use(Operand::Copy(p) | Operand::Move(p)))) = &stmt.kind { place.as_local().is_some() && p.as_local().is_some() @@ -147,8 +147,8 @@ fn get_arm_identity_info<'a, 'tcx>( /// Eats consecutive `Assign` Statements. // The iterator `stmt_iter` is not advanced if none were found. - fn try_eat_assign_tmp_stmts<'a, 'tcx>( - stmt_iter: &mut StmtIter<'a, 'tcx>, + fn try_eat_assign_tmp_stmts( + stmt_iter: &mut StmtIter<'_, '_>, tmp_assigns: &mut Vec<(Local, Local)>, nop_stmts: &mut Vec<usize>, ) { @@ -163,9 +163,9 @@ fn get_arm_identity_info<'a, 'tcx>( }) } - fn find_storage_live_dead_stmts_for_local<'tcx>( + fn find_storage_live_dead_stmts_for_local( local: Local, - stmts: &[Statement<'tcx>], + stmts: &[Statement<'_>], ) -> Option<(usize, usize)> { trace!("looking for {:?}", local); let mut storage_live_stmt = None; @@ -452,14 +452,14 @@ struct LocalUseCounter { } impl LocalUseCounter { - fn get_local_uses<'tcx>(body: &Body<'tcx>) -> IndexVec<Local, usize> { + fn get_local_uses(body: &Body<'_>) -> IndexVec<Local, usize> { let mut counter = LocalUseCounter { local_uses: IndexVec::from_elem(0, &body.local_decls) }; counter.visit_body(body); counter.local_uses } } -impl<'tcx> Visitor<'tcx> for LocalUseCounter { +impl Visitor<'_> for LocalUseCounter { fn visit_local(&mut self, local: &Local, context: PlaceContext, _location: Location) { if context.is_storage_marker() || context == PlaceContext::NonUse(NonUseContext::VarDebugInfo) @@ -510,7 +510,7 @@ fn match_set_variant_field<'tcx>(stmt: &Statement<'tcx>) -> Option<(Local, Local /// ```rust /// discriminant(_LOCAL_TO_SET) = VAR_IDX; /// ``` -fn match_set_discr<'tcx>(stmt: &Statement<'tcx>) -> Option<(Local, VariantIdx)> { +fn match_set_discr(stmt: &Statement<'_>) -> Option<(Local, VariantIdx)> { match &stmt.kind { StatementKind::SetDiscriminant { place, variant_index } => { Some((place.as_local()?, *variant_index)) @@ -588,7 +588,7 @@ struct SimplifyBranchSameOptimizationFinder<'a, 'tcx> { tcx: TyCtxt<'tcx>, } -impl<'a, 'tcx> SimplifyBranchSameOptimizationFinder<'a, 'tcx> { +impl<'tcx> SimplifyBranchSameOptimizationFinder<'_, 'tcx> { fn find(&self) -> Vec<SimplifyBranchSameOptimization> { self.body .basic_blocks() @@ -631,10 +631,6 @@ impl<'a, 'tcx> SimplifyBranchSameOptimizationFinder<'a, 'tcx> { .filter(|(_, bb)| { // Reaching `unreachable` is UB so assume it doesn't happen. bb.terminator().kind != TerminatorKind::Unreachable - // But `asm!(...)` could abort the program, - // so we cannot assume that the `unreachable` terminator itself is reachable. - // FIXME(Centril): use a normalization pass instead of a check. - || bb.statements.iter().any(|stmt| matches!(stmt.kind, StatementKind::LlvmInlineAsm(..))) }) .peekable(); diff --git a/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs b/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs index 2aa50611290..cda9ba9dcc8 100644 --- a/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs +++ b/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs @@ -3,8 +3,7 @@ use crate::MirPass; use rustc_data_structures::stable_set::FxHashSet; use rustc_middle::mir::{ - BasicBlock, BasicBlockData, Body, Local, Operand, Rvalue, StatementKind, SwitchTargets, - TerminatorKind, + BasicBlockData, Body, Local, Operand, Rvalue, StatementKind, SwitchTargets, TerminatorKind, }; use rustc_middle::ty::layout::TyAndLayout; use rustc_middle::ty::{Ty, TyCtxt}; @@ -56,7 +55,10 @@ fn variant_discriminants<'tcx>( match &layout.variants { Variants::Single { index } => { let mut res = FxHashSet::default(); - res.insert(index.as_u32() as u128); + res.insert( + ty.discriminant_for_variant(tcx, *index) + .map_or(index.as_u32() as u128, |discr| discr.val), + ); res } Variants::Multiple { variants, .. } => variants @@ -70,17 +72,14 @@ fn variant_discriminants<'tcx>( } impl<'tcx> MirPass<'tcx> for UninhabitedEnumBranching { - fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if body.source.promoted.is_some() { - return; - } + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() > 0 + } + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { trace!("UninhabitedEnumBranching starting for {:?}", body.source); - let basic_block_count = body.basic_blocks().len(); - - for bb in 0..basic_block_count { - let bb = BasicBlock::from_usize(bb); + for bb in body.basic_blocks().indices() { trace!("processing block {:?}", bb); let Some(discriminant_ty) = get_switched_on_type(&body.basic_blocks()[bb], tcx, body) else { diff --git a/compiler/rustc_mir_transform/src/unreachable_prop.rs b/compiler/rustc_mir_transform/src/unreachable_prop.rs index 64cd6f56a9f..f916ca36217 100644 --- a/compiler/rustc_mir_transform/src/unreachable_prop.rs +++ b/compiler/rustc_mir_transform/src/unreachable_prop.rs @@ -11,35 +11,26 @@ use rustc_middle::ty::TyCtxt; pub struct UnreachablePropagation; impl MirPass<'_> for UnreachablePropagation { - fn run_pass<'tcx>(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - if tcx.sess.mir_opt_level() < 4 { - // Enable only under -Zmir-opt-level=4 as in some cases (check the deeply-nested-opt - // perf benchmark) LLVM may spend quite a lot of time optimizing the generated code. - return; - } + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + // Enable only under -Zmir-opt-level=4 as in some cases (check the deeply-nested-opt + // perf benchmark) LLVM may spend quite a lot of time optimizing the generated code. + sess.mir_opt_level() >= 4 + } + fn run_pass<'tcx>(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let mut unreachable_blocks = FxHashSet::default(); let mut replacements = FxHashMap::default(); for (bb, bb_data) in traversal::postorder(body) { let terminator = bb_data.terminator(); - // HACK: If the block contains any asm statement it is not regarded as unreachable. - // This is a temporary solution that handles possibly diverging asm statements. - // Accompanying testcases: mir-opt/unreachable_asm.rs and mir-opt/unreachable_asm_2.rs - let asm_stmt_in_block = || { - bb_data.statements.iter().any(|stmt: &Statement<'_>| { - matches!(stmt.kind, StatementKind::LlvmInlineAsm(..)) - }) - }; - - if terminator.kind == TerminatorKind::Unreachable && !asm_stmt_in_block() { + if terminator.kind == TerminatorKind::Unreachable { unreachable_blocks.insert(bb); } else { let is_unreachable = |succ: BasicBlock| unreachable_blocks.contains(&succ); let terminator_kind_opt = remove_successors(&terminator.kind, is_unreachable); if let Some(terminator_kind) = terminator_kind_opt { - if terminator_kind == TerminatorKind::Unreachable && !asm_stmt_in_block() { + if terminator_kind == TerminatorKind::Unreachable { unreachable_blocks.insert(bb); } replacements.insert(bb, terminator_kind); @@ -64,7 +55,7 @@ impl MirPass<'_> for UnreachablePropagation { } } -fn remove_successors<F>( +fn remove_successors<'tcx, F>( terminator_kind: &TerminatorKind<'tcx>, predicate: F, ) -> Option<TerminatorKind<'tcx>> |